solid_queue 0.3.4 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,6 @@ module SolidQueue::Processes
9
9
  end
10
10
 
11
11
  def supervised_by(process)
12
- self.mode = :supervised
13
12
  @supervisor = process
14
13
  end
15
14
 
@@ -19,11 +18,11 @@ module SolidQueue::Processes
19
18
  end
20
19
 
21
20
  def supervisor_went_away?
22
- supervised? && supervisor&.pid != ::Process.ppid
21
+ supervised? && supervisor.pid != ::Process.ppid
23
22
  end
24
23
 
25
24
  def supervised?
26
- mode.supervised?
25
+ supervisor.present?
27
26
  end
28
27
 
29
28
  def register_signal_handlers
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Supervisor::AsyncSupervisor < Supervisor
5
+ def initialize(*)
6
+ super
7
+ @threads = Concurrent::Map.new
8
+ end
9
+
10
+ def kind
11
+ "Supervisor(async)"
12
+ end
13
+
14
+ def stop
15
+ super
16
+ stop_threads
17
+ threads.clear
18
+
19
+ shutdown
20
+ end
21
+
22
+ private
23
+ attr_reader :threads
24
+
25
+ def start_process(configured_process)
26
+ configured_process.supervised_by process
27
+ configured_process.start
28
+
29
+ threads[configured_process.name] = configured_process
30
+ end
31
+
32
+ def stop_threads
33
+ stop_threads = threads.values.map do |thr|
34
+ Thread.new { thr.stop }
35
+ end
36
+
37
+ stop_threads.each { |thr| thr.join(SolidQueue.shutdown_timeout) }
38
+ end
39
+
40
+ def all_threads_terminated?
41
+ threads.values.none?(&:alive?)
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,108 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Supervisor::ForkSupervisor < Supervisor
5
+ include Signals, Pidfiled
6
+
7
+ def initialize(*)
8
+ super
9
+ @forks = {}
10
+ end
11
+
12
+ def kind
13
+ "Supervisor(fork)"
14
+ end
15
+
16
+ private
17
+ attr_reader :forks
18
+
19
+ def supervise
20
+ loop do
21
+ break if stopped?
22
+
23
+ procline "supervising #{forks.keys.join(", ")}"
24
+ process_signal_queue
25
+
26
+ unless stopped?
27
+ reap_and_replace_terminated_forks
28
+ interruptible_sleep(1.second)
29
+ end
30
+ end
31
+ ensure
32
+ shutdown
33
+ end
34
+
35
+ def start_process(configured_process)
36
+ configured_process.supervised_by process
37
+ configured_process.mode = :fork
38
+
39
+ pid = fork do
40
+ configured_process.start
41
+ end
42
+
43
+ forks[pid] = configured_process
44
+ end
45
+
46
+ def terminate_gracefully
47
+ SolidQueue.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do |payload|
48
+ term_forks
49
+
50
+ Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
51
+ reap_terminated_forks
52
+ end
53
+
54
+ unless all_forks_terminated?
55
+ payload[:shutdown_timeout_exceeded] = true
56
+ terminate_immediately
57
+ end
58
+ end
59
+ end
60
+
61
+ def terminate_immediately
62
+ SolidQueue.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do
63
+ quit_forks
64
+ end
65
+ end
66
+
67
+ def term_forks
68
+ signal_processes(forks.keys, :TERM)
69
+ end
70
+
71
+ def quit_forks
72
+ signal_processes(forks.keys, :QUIT)
73
+ end
74
+
75
+ def reap_and_replace_terminated_forks
76
+ loop do
77
+ pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
78
+ break unless pid
79
+
80
+ replace_fork(pid, status)
81
+ end
82
+ end
83
+
84
+ def reap_terminated_forks
85
+ loop do
86
+ pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
87
+ break unless pid
88
+
89
+ forks.delete(pid)
90
+ end
91
+ rescue SystemCallError
92
+ # All children already reaped
93
+ end
94
+
95
+ def replace_fork(pid, status)
96
+ SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
97
+ if supervised_fork = forks.delete(pid)
98
+ payload[:fork] = supervised_fork
99
+ start_process(supervised_fork)
100
+ end
101
+ end
102
+ end
103
+
104
+ def all_forks_terminated?
105
+ forks.empty?
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,34 @@
1
+ module SolidQueue
2
+ module Supervisor::Maintenance
3
+ extend ActiveSupport::Concern
4
+
5
+ included do
6
+ after_boot :release_orphaned_executions
7
+ end
8
+
9
+ private
10
+ def launch_maintenance_task
11
+ @maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) do
12
+ prune_dead_processes
13
+ end
14
+
15
+ @maintenance_task.add_observer do |_, _, error|
16
+ handle_thread_error(error) if error
17
+ end
18
+
19
+ @maintenance_task.execute
20
+ end
21
+
22
+ def stop_maintenance_task
23
+ @maintenance_task&.shutdown
24
+ end
25
+
26
+ def prune_dead_processes
27
+ wrap_in_app_executor { SolidQueue::Process.prune }
28
+ end
29
+
30
+ def release_orphaned_executions
31
+ wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
32
+ end
33
+ end
34
+ end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module SolidQueue::Processes
4
- class Pidfile
3
+ module SolidQueue
4
+ class Supervisor::Pidfile
5
5
  def initialize(path)
6
6
  @path = path
7
7
  @pid = ::Process.pid
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Supervisor
5
+ module Pidfiled
6
+ extend ActiveSupport::Concern
7
+
8
+ included do
9
+ before_boot :setup_pidfile
10
+ after_shutdown :delete_pidfile
11
+ end
12
+
13
+ private
14
+ def setup_pidfile
15
+ if path = SolidQueue.supervisor_pidfile
16
+ @pidfile = Pidfile.new(path).tap(&:setup)
17
+ end
18
+ end
19
+
20
+ def delete_pidfile
21
+ @pidfile&.delete
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Supervisor
5
+ module Signals
6
+ extend ActiveSupport::Concern
7
+
8
+ included do
9
+ before_boot :register_signal_handlers
10
+ after_shutdown :restore_default_signal_handlers
11
+ end
12
+
13
+ private
14
+ SIGNALS = %i[ QUIT INT TERM ]
15
+
16
+ def register_signal_handlers
17
+ SIGNALS.each do |signal|
18
+ trap(signal) do
19
+ signal_queue << signal
20
+ interrupt
21
+ end
22
+ end
23
+ end
24
+
25
+ def restore_default_signal_handlers
26
+ SIGNALS.each do |signal|
27
+ trap(signal, :DEFAULT)
28
+ end
29
+ end
30
+
31
+ def process_signal_queue
32
+ while signal = signal_queue.shift
33
+ handle_signal(signal)
34
+ end
35
+ end
36
+
37
+ def handle_signal(signal)
38
+ case signal
39
+ when :TERM, :INT
40
+ stop
41
+ terminate_gracefully
42
+ when :QUIT
43
+ stop
44
+ terminate_immediately
45
+ else
46
+ SolidQueue.instrument :unhandled_signal_error, signal: signal
47
+ end
48
+ end
49
+
50
+ def signal_processes(pids, signal)
51
+ pids.each do |pid|
52
+ signal_process pid, signal
53
+ end
54
+ end
55
+
56
+ def signal_process(pid, signal)
57
+ ::Process.kill signal, pid
58
+ rescue Errno::ESRCH
59
+ # Ignore, process died before
60
+ end
61
+
62
+ def signal_queue
63
+ @signal_queue ||= []
64
+ end
65
+ end
66
+ end
67
+ end
@@ -2,190 +2,72 @@
2
2
 
3
3
  module SolidQueue
4
4
  class Supervisor < Processes::Base
5
- include Processes::Signals
5
+ include Maintenance
6
6
 
7
7
  class << self
8
- def start(mode: :work, load_configuration_from: nil)
8
+ def start(mode: :fork, load_configuration_from: nil)
9
9
  SolidQueue.supervisor = true
10
10
  configuration = Configuration.new(mode: mode, load_from: load_configuration_from)
11
11
 
12
- new(*configuration.processes).start
12
+ klass = mode == :fork ? ForkSupervisor : AsyncSupervisor
13
+ klass.new(configuration).tap(&:start)
13
14
  end
14
15
  end
15
16
 
16
- def initialize(*configured_processes)
17
- @configured_processes = Array(configured_processes)
18
- @forks = {}
17
+ def initialize(configuration)
18
+ @configuration = configuration
19
19
  end
20
20
 
21
21
  def start
22
- run_callbacks(:boot) { boot }
22
+ boot
23
23
 
24
- start_forks
24
+ start_processes
25
25
  launch_maintenance_task
26
26
 
27
27
  supervise
28
- rescue Processes::GracefulTerminationRequested
29
- graceful_termination
30
- rescue Processes::ImmediateTerminationRequested
31
- immediate_termination
32
- ensure
33
- run_callbacks(:shutdown) { shutdown }
28
+ end
29
+
30
+ def stop
31
+ @stopped = true
34
32
  end
35
33
 
36
34
  private
37
- attr_reader :configured_processes, :forks
35
+ attr_reader :configuration
38
36
 
39
37
  def boot
40
- sync_std_streams
41
- setup_pidfile
42
- register_signal_handlers
43
- end
44
-
45
- def supervise
46
- loop do
47
- procline "supervising #{forks.keys.join(", ")}"
48
-
49
- process_signal_queue
50
- reap_and_replace_terminated_forks
51
- interruptible_sleep(1.second)
52
- end
53
- end
54
-
55
- def sync_std_streams
56
- STDOUT.sync = STDERR.sync = true
57
- end
58
-
59
- def setup_pidfile
60
- @pidfile = if SolidQueue.supervisor_pidfile
61
- Processes::Pidfile.new(SolidQueue.supervisor_pidfile).tap(&:setup)
62
- end
63
- end
64
-
65
- def start_forks
66
- configured_processes.each { |configured_process| start_fork(configured_process) }
67
- end
68
-
69
- def launch_maintenance_task
70
- @maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) do
71
- prune_dead_processes
72
- release_orphaned_executions
73
- end
74
- @maintenance_task.execute
75
- end
76
-
77
- def shutdown
78
- stop_process_prune
79
- restore_default_signal_handlers
80
- delete_pidfile
81
- end
82
-
83
- def graceful_termination
84
- SolidQueue.instrument(:graceful_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do |payload|
85
- term_forks
86
-
87
- wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
88
- reap_terminated_forks
89
- end
90
-
91
- unless all_forks_terminated?
92
- payload[:shutdown_timeout_exceeded] = true
93
- immediate_termination
38
+ SolidQueue.instrument(:start_process, process: self) do
39
+ run_callbacks(:boot) do
40
+ @stopped = false
41
+ sync_std_streams
94
42
  end
95
43
  end
96
44
  end
97
45
 
98
- def immediate_termination
99
- SolidQueue.instrument(:immediate_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do
100
- quit_forks
101
- end
102
- end
103
-
104
- def term_forks
105
- signal_processes(forks.keys, :TERM)
46
+ def start_processes
47
+ configuration.processes.each { |configured_process| start_process(configured_process) }
106
48
  end
107
49
 
108
- def quit_forks
109
- signal_processes(forks.keys, :QUIT)
50
+ def stopped?
51
+ @stopped
110
52
  end
111
53
 
112
- def stop_process_prune
113
- @maintenance_task&.shutdown
114
- end
115
-
116
- def delete_pidfile
117
- @pidfile&.delete
118
- end
119
-
120
- def prune_dead_processes
121
- wrap_in_app_executor { SolidQueue::Process.prune }
122
- end
123
-
124
- def release_orphaned_executions
125
- wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
126
- end
127
-
128
- def start_fork(configured_process)
129
- configured_process.supervised_by process
130
-
131
- pid = fork do
132
- configured_process.start
133
- end
134
-
135
- forks[pid] = configured_process
136
- end
137
-
138
- def reap_and_replace_terminated_forks
139
- loop do
140
- pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
141
- break unless pid
142
-
143
- replace_fork(pid, status)
144
- end
145
- end
146
-
147
- def reap_terminated_forks
148
- loop do
149
- pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
150
- break unless pid
151
-
152
- forks.delete(pid)
153
- end
154
- rescue SystemCallError
155
- # All children already reaped
156
- end
157
-
158
- def replace_fork(pid, status)
159
- SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
160
- if supervised_fork = forks.delete(pid)
161
- payload[:fork] = supervised_fork
162
- start_fork(supervised_fork)
163
- end
164
- end
54
+ def supervise
165
55
  end
166
56
 
167
- def all_forks_terminated?
168
- forks.empty?
57
+ def start_process(configured_process)
58
+ raise NotImplementedError
169
59
  end
170
60
 
171
- def wait_until(timeout, condition, &block)
172
- if timeout > 0
173
- deadline = monotonic_time_now + timeout
174
-
175
- while monotonic_time_now < deadline && !condition.call
176
- sleep 0.1
177
- block.call
178
- end
179
- else
180
- while !condition.call
181
- sleep 0.5
182
- block.call
61
+ def shutdown
62
+ SolidQueue.instrument(:shutdown_process, process: self) do
63
+ run_callbacks(:shutdown) do
64
+ stop_maintenance_task
183
65
  end
184
66
  end
185
67
  end
186
68
 
187
- def monotonic_time_now
188
- ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
69
+ def sync_std_streams
70
+ STDOUT.sync = STDERR.sync = true
189
71
  end
190
72
  end
191
73
  end
@@ -1,16 +1,6 @@
1
1
  namespace :solid_queue do
2
2
  desc "start solid_queue supervisor to dispatch and process jobs"
3
3
  task start: :environment do
4
- SolidQueue::Supervisor.start(mode: :all)
5
- end
6
-
7
- desc "start solid_queue supervisor to process jobs"
8
- task work: :environment do
9
- SolidQueue::Supervisor.start(mode: :work)
10
- end
11
-
12
- desc "start solid_queue dispatcher to enqueue scheduled jobs"
13
- task dispatch: :environment do
14
- SolidQueue::Supervisor.start(mode: :dispatch)
4
+ SolidQueue::Supervisor.start
15
5
  end
16
6
  end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ module Timer
5
+ extend self
6
+
7
+ def wait_until(timeout, condition, &block)
8
+ if timeout > 0
9
+ deadline = monotonic_time_now + timeout
10
+
11
+ while monotonic_time_now < deadline && !condition.call
12
+ sleep 0.1
13
+ block.call
14
+ end
15
+ else
16
+ while !condition.call
17
+ sleep 0.5
18
+ block.call
19
+ end
20
+ end
21
+ end
22
+
23
+ private
24
+ def monotonic_time_now
25
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
26
+ end
27
+ end
28
+ end
@@ -1,3 +1,3 @@
1
1
  module SolidQueue
2
- VERSION = "0.3.4"
2
+ VERSION = "0.4.1"
3
3
  end
@@ -1,17 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SolidQueue
4
- class Worker < Processes::Base
5
- include Processes::Poller
6
-
4
+ class Worker < Processes::Poller
7
5
  attr_accessor :queues, :pool
8
6
 
9
7
  def initialize(**options)
10
8
  options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
11
9
 
12
- @polling_interval = options[:polling_interval]
13
10
  @queues = Array(options[:queues])
14
11
  @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
12
+
13
+ super(**options)
15
14
  end
16
15
 
17
16
  def metadata
@@ -31,7 +30,7 @@ module SolidQueue
31
30
 
32
31
  def claim_executions
33
32
  with_polling_volume do
34
- SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process.id)
33
+ SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process_id)
35
34
  end
36
35
  end
37
36
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: solid_queue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rosa Gutierrez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-07-19 00:00:00.000000000 Z
11
+ date: 2024-08-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -203,6 +203,7 @@ files:
203
203
  - app/models/solid_queue/job/schedulable.rb
204
204
  - app/models/solid_queue/pause.rb
205
205
  - app/models/solid_queue/process.rb
206
+ - app/models/solid_queue/process/executor.rb
206
207
  - app/models/solid_queue/process/prunable.rb
207
208
  - app/models/solid_queue/queue.rb
208
209
  - app/models/solid_queue/queue_selector.rb
@@ -234,15 +235,20 @@ files:
234
235
  - lib/solid_queue/processes/base.rb
235
236
  - lib/solid_queue/processes/callbacks.rb
236
237
  - lib/solid_queue/processes/interruptible.rb
237
- - lib/solid_queue/processes/pidfile.rb
238
238
  - lib/solid_queue/processes/poller.rb
239
239
  - lib/solid_queue/processes/procline.rb
240
240
  - lib/solid_queue/processes/registrable.rb
241
241
  - lib/solid_queue/processes/runnable.rb
242
- - lib/solid_queue/processes/signals.rb
243
242
  - lib/solid_queue/processes/supervised.rb
244
243
  - lib/solid_queue/supervisor.rb
244
+ - lib/solid_queue/supervisor/async_supervisor.rb
245
+ - lib/solid_queue/supervisor/fork_supervisor.rb
246
+ - lib/solid_queue/supervisor/maintenance.rb
247
+ - lib/solid_queue/supervisor/pidfile.rb
248
+ - lib/solid_queue/supervisor/pidfiled.rb
249
+ - lib/solid_queue/supervisor/signals.rb
245
250
  - lib/solid_queue/tasks.rb
251
+ - lib/solid_queue/timer.rb
246
252
  - lib/solid_queue/version.rb
247
253
  - lib/solid_queue/worker.rb
248
254
  homepage: https://github.com/rails/solid_queue
@@ -251,7 +257,9 @@ licenses:
251
257
  metadata:
252
258
  homepage_uri: https://github.com/rails/solid_queue
253
259
  source_code_uri: https://github.com/rails/solid_queue
254
- post_install_message:
260
+ post_install_message: |
261
+ Upgrading to Solid Queue 0.4.x? There are some breaking changes about how Solid Queue is started. Check
262
+ https://github.com/rails/solid_queue/blob/main/UPGRADING.md for upgrade instructions.
255
263
  rdoc_options: []
256
264
  require_paths:
257
265
  - lib
@@ -266,7 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
266
274
  - !ruby/object:Gem::Version
267
275
  version: '0'
268
276
  requirements: []
269
- rubygems_version: 3.5.16
277
+ rubygems_version: 3.5.9
270
278
  signing_key:
271
279
  specification_version: 4
272
280
  summary: Database-backed Active Job backend.