solid_queue 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/MIT-LICENSE +20 -0
- data/README.md +230 -0
- data/Rakefile +8 -0
- data/app/models/solid_queue/blocked_execution.rb +68 -0
- data/app/models/solid_queue/claimed_execution.rb +73 -0
- data/app/models/solid_queue/execution/job_attributes.rb +24 -0
- data/app/models/solid_queue/execution.rb +15 -0
- data/app/models/solid_queue/failed_execution.rb +31 -0
- data/app/models/solid_queue/job/clearable.rb +19 -0
- data/app/models/solid_queue/job/concurrency_controls.rb +50 -0
- data/app/models/solid_queue/job/executable.rb +87 -0
- data/app/models/solid_queue/job.rb +38 -0
- data/app/models/solid_queue/pause.rb +6 -0
- data/app/models/solid_queue/process/prunable.rb +20 -0
- data/app/models/solid_queue/process.rb +28 -0
- data/app/models/solid_queue/queue.rb +52 -0
- data/app/models/solid_queue/queue_selector.rb +68 -0
- data/app/models/solid_queue/ready_execution.rb +41 -0
- data/app/models/solid_queue/record.rb +19 -0
- data/app/models/solid_queue/scheduled_execution.rb +65 -0
- data/app/models/solid_queue/semaphore.rb +65 -0
- data/config/routes.rb +2 -0
- data/db/migrate/20231211200639_create_solid_queue_tables.rb +100 -0
- data/lib/active_job/concurrency_controls.rb +53 -0
- data/lib/active_job/queue_adapters/solid_queue_adapter.rb +24 -0
- data/lib/generators/solid_queue/install/USAGE +9 -0
- data/lib/generators/solid_queue/install/install_generator.rb +19 -0
- data/lib/puma/plugin/solid_queue.rb +63 -0
- data/lib/solid_queue/app_executor.rb +21 -0
- data/lib/solid_queue/configuration.rb +102 -0
- data/lib/solid_queue/dispatcher.rb +73 -0
- data/lib/solid_queue/engine.rb +39 -0
- data/lib/solid_queue/pool.rb +58 -0
- data/lib/solid_queue/processes/base.rb +27 -0
- data/lib/solid_queue/processes/interruptible.rb +37 -0
- data/lib/solid_queue/processes/pidfile.rb +58 -0
- data/lib/solid_queue/processes/poller.rb +24 -0
- data/lib/solid_queue/processes/procline.rb +11 -0
- data/lib/solid_queue/processes/registrable.rb +69 -0
- data/lib/solid_queue/processes/runnable.rb +77 -0
- data/lib/solid_queue/processes/signals.rb +69 -0
- data/lib/solid_queue/processes/supervised.rb +38 -0
- data/lib/solid_queue/supervisor.rb +182 -0
- data/lib/solid_queue/tasks.rb +16 -0
- data/lib/solid_queue/version.rb +3 -0
- data/lib/solid_queue/worker.rb +54 -0
- data/lib/solid_queue.rb +52 -0
- metadata +134 -0
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
module AppExecutor
|
5
|
+
def wrap_in_app_executor(&block)
|
6
|
+
if SolidQueue.app_executor
|
7
|
+
SolidQueue.app_executor.wrap(&block)
|
8
|
+
else
|
9
|
+
yield
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def handle_thread_error(error)
|
14
|
+
SolidQueue.logger.error("[SolidQueue] #{error}")
|
15
|
+
|
16
|
+
if SolidQueue.on_thread_error
|
17
|
+
SolidQueue.on_thread_error.call(error)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
class Configuration
|
5
|
+
WORKER_DEFAULTS = {
|
6
|
+
queues: "*",
|
7
|
+
threads: 5,
|
8
|
+
processes: 1,
|
9
|
+
polling_interval: 0.1
|
10
|
+
}
|
11
|
+
|
12
|
+
DISPATCHER_DEFAULTS = {
|
13
|
+
batch_size: 500,
|
14
|
+
polling_interval: 1,
|
15
|
+
concurrency_maintenance_interval: 600
|
16
|
+
}
|
17
|
+
|
18
|
+
def initialize(mode: :work, load_from: nil)
|
19
|
+
@mode = mode
|
20
|
+
@raw_config = config_from(load_from)
|
21
|
+
end
|
22
|
+
|
23
|
+
def processes
|
24
|
+
case mode
|
25
|
+
when :dispatch then dispatchers
|
26
|
+
when :work then workers
|
27
|
+
when :all then dispatchers + workers
|
28
|
+
else raise "Invalid mode #{mode}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def workers
|
33
|
+
if mode.in? %i[ work all]
|
34
|
+
workers_options.flat_map do |worker_options|
|
35
|
+
processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
|
36
|
+
processes.times.collect { SolidQueue::Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
|
37
|
+
end
|
38
|
+
else
|
39
|
+
[]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def dispatchers
|
44
|
+
if mode.in? %i[ dispatch all]
|
45
|
+
dispatchers_options.flat_map do |dispatcher_options|
|
46
|
+
SolidQueue::Dispatcher.new(**dispatcher_options)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def max_number_of_threads
|
52
|
+
# At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
|
53
|
+
workers_options.map { |options| options[:threads] }.max + 2
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
attr_reader :raw_config, :mode
|
58
|
+
|
59
|
+
DEFAULT_CONFIG_FILE_PATH = "config/solid_queue.yml"
|
60
|
+
|
61
|
+
def config_from(file_or_hash, env: Rails.env)
|
62
|
+
config = load_config_from(file_or_hash)
|
63
|
+
config[env.to_sym] ? config[env.to_sym] : config
|
64
|
+
end
|
65
|
+
|
66
|
+
def workers_options
|
67
|
+
@workers_options ||= (raw_config[:workers] || [ WORKER_DEFAULTS ])
|
68
|
+
.map { |options| options.dup.symbolize_keys }
|
69
|
+
end
|
70
|
+
|
71
|
+
def dispatchers_options
|
72
|
+
@dispatchers_options ||= (raw_config[:dispatchers] || [ DISPATCHER_DEFAULTS ])
|
73
|
+
.map { |options| options.dup.symbolize_keys }
|
74
|
+
end
|
75
|
+
|
76
|
+
def load_config_from(file_or_hash)
|
77
|
+
case file_or_hash
|
78
|
+
when Pathname then load_config_file file_or_hash
|
79
|
+
when String then load_config_file Pathname.new(file_or_hash)
|
80
|
+
when NilClass then load_config_file default_config_file
|
81
|
+
when Hash then file_or_hash.dup
|
82
|
+
else raise "Solid Queue cannot be initialized with #{file_or_hash.inspect}"
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def load_config_file(file)
|
87
|
+
if file.exist?
|
88
|
+
ActiveSupport::ConfigurationFile.parse(file).deep_symbolize_keys
|
89
|
+
else
|
90
|
+
raise "Configuration file not found in #{file}"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def default_config_file
|
95
|
+
path_to_file = ENV["SOLID_QUEUE_CONFIG"] || DEFAULT_CONFIG_FILE_PATH
|
96
|
+
|
97
|
+
Rails.root.join(path_to_file).tap do |config_file|
|
98
|
+
raise "Configuration for Solid Queue not found in #{config_file}" unless config_file.exist?
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
class Dispatcher < Processes::Base
|
5
|
+
include Processes::Runnable, Processes::Poller
|
6
|
+
|
7
|
+
attr_accessor :batch_size, :concurrency_maintenance_interval
|
8
|
+
|
9
|
+
set_callback :boot, :after, :launch_concurrency_maintenance
|
10
|
+
set_callback :shutdown, :before, :stop_concurrency_maintenance
|
11
|
+
|
12
|
+
def initialize(**options)
|
13
|
+
options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
|
14
|
+
|
15
|
+
@batch_size = options[:batch_size]
|
16
|
+
@polling_interval = options[:polling_interval]
|
17
|
+
@concurrency_maintenance_interval = options[:concurrency_maintenance_interval]
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
def run
|
22
|
+
batch = dispatch_next_batch
|
23
|
+
|
24
|
+
unless batch.size > 0
|
25
|
+
procline "waiting"
|
26
|
+
interruptible_sleep(polling_interval)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def dispatch_next_batch
|
31
|
+
with_polling_volume do
|
32
|
+
SolidQueue::ScheduledExecution.dispatch_next_batch(batch_size)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def launch_concurrency_maintenance
|
37
|
+
@concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: concurrency_maintenance_interval) do
|
38
|
+
expire_semaphores
|
39
|
+
unblock_blocked_executions
|
40
|
+
end
|
41
|
+
|
42
|
+
@concurrency_maintenance_task.add_observer do |_, _, error|
|
43
|
+
handle_thread_error(error) if error
|
44
|
+
end
|
45
|
+
|
46
|
+
@concurrency_maintenance_task.execute
|
47
|
+
end
|
48
|
+
|
49
|
+
def stop_concurrency_maintenance
|
50
|
+
@concurrency_maintenance_task.shutdown
|
51
|
+
end
|
52
|
+
|
53
|
+
def expire_semaphores
|
54
|
+
wrap_in_app_executor do
|
55
|
+
Semaphore.expired.in_batches(of: batch_size, &:delete_all)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def unblock_blocked_executions
|
60
|
+
wrap_in_app_executor do
|
61
|
+
BlockedExecution.unblock(batch_size)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def initial_jitter
|
66
|
+
Kernel.rand(0...polling_interval)
|
67
|
+
end
|
68
|
+
|
69
|
+
def metadata
|
70
|
+
super.merge(batch_size: batch_size)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
class Engine < ::Rails::Engine
|
5
|
+
isolate_namespace SolidQueue
|
6
|
+
|
7
|
+
rake_tasks do
|
8
|
+
load "solid_queue/tasks.rb"
|
9
|
+
end
|
10
|
+
|
11
|
+
config.solid_queue = ActiveSupport::OrderedOptions.new
|
12
|
+
|
13
|
+
initializer "solid_queue.config" do
|
14
|
+
config.solid_queue.each do |name, value|
|
15
|
+
SolidQueue.public_send("#{name}=", value)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
initializer "solid_queue.app_executor", before: :run_prepare_callbacks do |app|
|
20
|
+
config.solid_queue.app_executor ||= app.executor
|
21
|
+
config.solid_queue.on_thread_error ||= -> (exception) { Rails.error.report(exception, handled: false) }
|
22
|
+
|
23
|
+
SolidQueue.app_executor = config.solid_queue.app_executor
|
24
|
+
SolidQueue.on_thread_error = config.solid_queue.on_thread_error
|
25
|
+
end
|
26
|
+
|
27
|
+
initializer "solid_queue.logger" do |app|
|
28
|
+
ActiveSupport.on_load(:solid_queue) do
|
29
|
+
self.logger = app.logger
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
initializer "solid_queue.active_job.extensions" do
|
34
|
+
ActiveSupport.on_load :active_job do
|
35
|
+
include ActiveJob::ConcurrencyControls
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
class Pool
|
5
|
+
include AppExecutor
|
6
|
+
|
7
|
+
attr_reader :size
|
8
|
+
|
9
|
+
delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor
|
10
|
+
|
11
|
+
def initialize(size, on_idle: nil)
|
12
|
+
@size = size
|
13
|
+
@on_idle = on_idle
|
14
|
+
@available_threads = Concurrent::AtomicFixnum.new(size)
|
15
|
+
@mutex = Mutex.new
|
16
|
+
end
|
17
|
+
|
18
|
+
def post(execution)
|
19
|
+
available_threads.decrement
|
20
|
+
|
21
|
+
future = Concurrent::Future.new(args: [ execution ], executor: executor) do |thread_execution|
|
22
|
+
wrap_in_app_executor do
|
23
|
+
thread_execution.perform
|
24
|
+
ensure
|
25
|
+
available_threads.increment
|
26
|
+
mutex.synchronize { on_idle.try(:call) if idle? }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
future.add_observer do |_, _, error|
|
31
|
+
handle_thread_error(error) if error
|
32
|
+
end
|
33
|
+
|
34
|
+
future.execute
|
35
|
+
end
|
36
|
+
|
37
|
+
def idle_threads
|
38
|
+
available_threads.value
|
39
|
+
end
|
40
|
+
|
41
|
+
def idle?
|
42
|
+
idle_threads > 0
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
attr_reader :available_threads, :on_idle, :mutex
|
47
|
+
|
48
|
+
DEFAULT_OPTIONS = {
|
49
|
+
min_threads: 0,
|
50
|
+
idletime: 60,
|
51
|
+
fallback_policy: :abort
|
52
|
+
}
|
53
|
+
|
54
|
+
def executor
|
55
|
+
@executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
module Processes
|
5
|
+
class Base
|
6
|
+
include ActiveSupport::Callbacks
|
7
|
+
define_callbacks :boot, :shutdown
|
8
|
+
|
9
|
+
include AppExecutor, Registrable, Interruptible, Procline
|
10
|
+
|
11
|
+
private
|
12
|
+
def observe_initial_delay
|
13
|
+
interruptible_sleep(initial_jitter)
|
14
|
+
end
|
15
|
+
|
16
|
+
def boot
|
17
|
+
end
|
18
|
+
|
19
|
+
def shutdown
|
20
|
+
end
|
21
|
+
|
22
|
+
def initial_jitter
|
23
|
+
0
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue::Processes
|
4
|
+
module Interruptible
|
5
|
+
def wake_up
|
6
|
+
interrupt
|
7
|
+
end
|
8
|
+
|
9
|
+
private
|
10
|
+
SELF_PIPE_BLOCK_SIZE = 11
|
11
|
+
|
12
|
+
def interrupt
|
13
|
+
self_pipe[:writer].write_nonblock( "." )
|
14
|
+
rescue Errno::EAGAIN, Errno::EINTR
|
15
|
+
# Ignore writes that would block and retry
|
16
|
+
# if another signal arrived while writing
|
17
|
+
retry
|
18
|
+
end
|
19
|
+
|
20
|
+
def interruptible_sleep(time)
|
21
|
+
if time > 0 && self_pipe[:reader].wait_readable(time)
|
22
|
+
loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) }
|
23
|
+
end
|
24
|
+
rescue Errno::EAGAIN, Errno::EINTR
|
25
|
+
end
|
26
|
+
|
27
|
+
# Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html)
|
28
|
+
def self_pipe
|
29
|
+
@self_pipe ||= create_self_pipe
|
30
|
+
end
|
31
|
+
|
32
|
+
def create_self_pipe
|
33
|
+
reader, writer = IO.pipe
|
34
|
+
{ reader: reader, writer: writer }
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue::Processes
|
4
|
+
class Pidfile
|
5
|
+
def initialize(path)
|
6
|
+
@path = path
|
7
|
+
@pid = ::Process.pid
|
8
|
+
end
|
9
|
+
|
10
|
+
def setup
|
11
|
+
check_status
|
12
|
+
write_file
|
13
|
+
set_at_exit_hook
|
14
|
+
end
|
15
|
+
|
16
|
+
def delete
|
17
|
+
delete_file
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
attr_reader :path, :pid
|
22
|
+
|
23
|
+
def check_status
|
24
|
+
if ::File.exist?(path)
|
25
|
+
existing_pid = ::File.open(path).read.strip.to_i
|
26
|
+
existing_pid > 0 && ::Process.kill(0, existing_pid)
|
27
|
+
|
28
|
+
already_running!
|
29
|
+
else
|
30
|
+
FileUtils.mkdir_p File.dirname(path)
|
31
|
+
end
|
32
|
+
rescue Errno::ESRCH => e
|
33
|
+
# Process is dead, ignore, just delete the file
|
34
|
+
delete
|
35
|
+
rescue Errno::EPERM
|
36
|
+
already_running!
|
37
|
+
end
|
38
|
+
|
39
|
+
def write_file
|
40
|
+
::File.open(path, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |file| file.write(pid.to_s) }
|
41
|
+
rescue Errno::EEXIST
|
42
|
+
check_status
|
43
|
+
retry
|
44
|
+
end
|
45
|
+
|
46
|
+
def set_at_exit_hook
|
47
|
+
at_exit { delete if ::Process.pid == pid }
|
48
|
+
end
|
49
|
+
|
50
|
+
def delete_file
|
51
|
+
::File.delete(path) if ::File.exist?(path)
|
52
|
+
end
|
53
|
+
|
54
|
+
def already_running!
|
55
|
+
abort "A Solid Queue supervisor is already running. Check #{path}"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue::Processes
|
4
|
+
module Poller
|
5
|
+
extend ActiveSupport::Concern
|
6
|
+
|
7
|
+
included do
|
8
|
+
attr_accessor :polling_interval
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
def with_polling_volume
|
13
|
+
if SolidQueue.silence_polling?
|
14
|
+
ActiveRecord::Base.logger.silence { yield }
|
15
|
+
else
|
16
|
+
yield
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def metadata
|
21
|
+
super.merge(polling_interval: polling_interval)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue::Processes
|
4
|
+
module Procline
|
5
|
+
# Sets the procline ($0)
|
6
|
+
# solid-queue-supervisor(0.1.0): <string>
|
7
|
+
def procline(string)
|
8
|
+
$0 = "solid-queue-#{self.class.name.demodulize.downcase}(#{SolidQueue::VERSION}): #{string}"
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue::Processes
|
4
|
+
module Registrable
|
5
|
+
extend ActiveSupport::Concern
|
6
|
+
|
7
|
+
included do
|
8
|
+
set_callback :boot, :after, :register
|
9
|
+
set_callback :boot, :after, :launch_heartbeat
|
10
|
+
|
11
|
+
set_callback :shutdown, :before, :stop_heartbeat
|
12
|
+
set_callback :shutdown, :after, :deregister
|
13
|
+
end
|
14
|
+
|
15
|
+
def inspect
|
16
|
+
"#{kind}(pid=#{process_pid}, hostname=#{hostname}, metadata=#{metadata})"
|
17
|
+
end
|
18
|
+
alias to_s inspect
|
19
|
+
|
20
|
+
private
|
21
|
+
attr_accessor :process
|
22
|
+
|
23
|
+
def register
|
24
|
+
@process = SolidQueue::Process.register \
|
25
|
+
kind: self.class.name.demodulize,
|
26
|
+
pid: process_pid,
|
27
|
+
hostname: hostname,
|
28
|
+
supervisor: try(:supervisor),
|
29
|
+
metadata: metadata
|
30
|
+
end
|
31
|
+
|
32
|
+
def deregister
|
33
|
+
process.deregister if registered?
|
34
|
+
end
|
35
|
+
|
36
|
+
def registered?
|
37
|
+
process&.persisted?
|
38
|
+
end
|
39
|
+
|
40
|
+
def launch_heartbeat
|
41
|
+
@heartbeat_task = Concurrent::TimerTask.new(execution_interval: SolidQueue.process_heartbeat_interval) { heartbeat }
|
42
|
+
@heartbeat_task.execute
|
43
|
+
end
|
44
|
+
|
45
|
+
def stop_heartbeat
|
46
|
+
@heartbeat_task&.shutdown
|
47
|
+
end
|
48
|
+
|
49
|
+
def heartbeat
|
50
|
+
process.heartbeat
|
51
|
+
end
|
52
|
+
|
53
|
+
def kind
|
54
|
+
self.class.name.demodulize
|
55
|
+
end
|
56
|
+
|
57
|
+
def hostname
|
58
|
+
@hostname ||= Socket.gethostname
|
59
|
+
end
|
60
|
+
|
61
|
+
def process_pid
|
62
|
+
@pid ||= ::Process.pid
|
63
|
+
end
|
64
|
+
|
65
|
+
def metadata
|
66
|
+
{}
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue::Processes
|
4
|
+
module Runnable
|
5
|
+
include Supervised
|
6
|
+
|
7
|
+
def start
|
8
|
+
@stopping = false
|
9
|
+
|
10
|
+
observe_initial_delay
|
11
|
+
run_callbacks(:boot) { boot }
|
12
|
+
|
13
|
+
start_loop
|
14
|
+
end
|
15
|
+
|
16
|
+
def stop
|
17
|
+
@stopping = true
|
18
|
+
@thread&.join
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
attr_writer :mode
|
23
|
+
|
24
|
+
DEFAULT_MODE = :async
|
25
|
+
|
26
|
+
def mode
|
27
|
+
(@mode || DEFAULT_MODE).to_s.inquiry
|
28
|
+
end
|
29
|
+
|
30
|
+
def boot
|
31
|
+
register_signal_handlers if supervised?
|
32
|
+
SolidQueue.logger.info("[SolidQueue] Starting #{self}")
|
33
|
+
end
|
34
|
+
|
35
|
+
def start_loop
|
36
|
+
if mode.async?
|
37
|
+
@thread = Thread.new { do_start_loop }
|
38
|
+
else
|
39
|
+
do_start_loop
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def do_start_loop
|
44
|
+
loop do
|
45
|
+
break if shutting_down?
|
46
|
+
|
47
|
+
run
|
48
|
+
end
|
49
|
+
ensure
|
50
|
+
run_callbacks(:shutdown) { shutdown }
|
51
|
+
end
|
52
|
+
|
53
|
+
def shutting_down?
|
54
|
+
stopping? || supervisor_went_away? || finished?
|
55
|
+
end
|
56
|
+
|
57
|
+
def run
|
58
|
+
raise NotImplementedError
|
59
|
+
end
|
60
|
+
|
61
|
+
def stopping?
|
62
|
+
@stopping
|
63
|
+
end
|
64
|
+
|
65
|
+
def finished?
|
66
|
+
running_inline? && all_work_completed?
|
67
|
+
end
|
68
|
+
|
69
|
+
def all_work_completed?
|
70
|
+
false
|
71
|
+
end
|
72
|
+
|
73
|
+
def running_inline?
|
74
|
+
mode.inline?
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue::Processes
|
4
|
+
class GracefulTerminationRequested < Interrupt; end
|
5
|
+
class ImmediateTerminationRequested < Interrupt; end
|
6
|
+
|
7
|
+
module Signals
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
private
|
11
|
+
SIGNALS = %i[ QUIT INT TERM ]
|
12
|
+
|
13
|
+
def register_signal_handlers
|
14
|
+
SIGNALS.each do |signal|
|
15
|
+
trap(signal) do
|
16
|
+
signal_queue << signal
|
17
|
+
interrupt
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def restore_default_signal_handlers
|
23
|
+
SIGNALS.each do |signal|
|
24
|
+
trap(signal, :DEFAULT)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def process_signal_queue
|
29
|
+
while signal = signal_queue.shift
|
30
|
+
handle_signal(signal)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def handle_signal(signal)
|
35
|
+
case signal
|
36
|
+
when :TERM, :INT
|
37
|
+
request_graceful_termination
|
38
|
+
when :QUIT
|
39
|
+
request_immediate_termination
|
40
|
+
else
|
41
|
+
SolidQueue.logger.warn "Received unhandled signal #{signal}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def request_graceful_termination
|
46
|
+
raise GracefulTerminationRequested
|
47
|
+
end
|
48
|
+
|
49
|
+
def request_immediate_termination
|
50
|
+
raise ImmediateTerminationRequested
|
51
|
+
end
|
52
|
+
|
53
|
+
def signal_processes(pids, signal)
|
54
|
+
pids.each do |pid|
|
55
|
+
signal_process pid, signal
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def signal_process(pid, signal)
|
60
|
+
::Process.kill signal, pid
|
61
|
+
rescue Errno::ESRCH
|
62
|
+
# Ignore, process died before
|
63
|
+
end
|
64
|
+
|
65
|
+
def signal_queue
|
66
|
+
@signal_queue ||= []
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|