solid_queue 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +20 -0
  3. data/README.md +230 -0
  4. data/Rakefile +8 -0
  5. data/app/models/solid_queue/blocked_execution.rb +68 -0
  6. data/app/models/solid_queue/claimed_execution.rb +73 -0
  7. data/app/models/solid_queue/execution/job_attributes.rb +24 -0
  8. data/app/models/solid_queue/execution.rb +15 -0
  9. data/app/models/solid_queue/failed_execution.rb +31 -0
  10. data/app/models/solid_queue/job/clearable.rb +19 -0
  11. data/app/models/solid_queue/job/concurrency_controls.rb +50 -0
  12. data/app/models/solid_queue/job/executable.rb +87 -0
  13. data/app/models/solid_queue/job.rb +38 -0
  14. data/app/models/solid_queue/pause.rb +6 -0
  15. data/app/models/solid_queue/process/prunable.rb +20 -0
  16. data/app/models/solid_queue/process.rb +28 -0
  17. data/app/models/solid_queue/queue.rb +52 -0
  18. data/app/models/solid_queue/queue_selector.rb +68 -0
  19. data/app/models/solid_queue/ready_execution.rb +41 -0
  20. data/app/models/solid_queue/record.rb +19 -0
  21. data/app/models/solid_queue/scheduled_execution.rb +65 -0
  22. data/app/models/solid_queue/semaphore.rb +65 -0
  23. data/config/routes.rb +2 -0
  24. data/db/migrate/20231211200639_create_solid_queue_tables.rb +100 -0
  25. data/lib/active_job/concurrency_controls.rb +53 -0
  26. data/lib/active_job/queue_adapters/solid_queue_adapter.rb +24 -0
  27. data/lib/generators/solid_queue/install/USAGE +9 -0
  28. data/lib/generators/solid_queue/install/install_generator.rb +19 -0
  29. data/lib/puma/plugin/solid_queue.rb +63 -0
  30. data/lib/solid_queue/app_executor.rb +21 -0
  31. data/lib/solid_queue/configuration.rb +102 -0
  32. data/lib/solid_queue/dispatcher.rb +73 -0
  33. data/lib/solid_queue/engine.rb +39 -0
  34. data/lib/solid_queue/pool.rb +58 -0
  35. data/lib/solid_queue/processes/base.rb +27 -0
  36. data/lib/solid_queue/processes/interruptible.rb +37 -0
  37. data/lib/solid_queue/processes/pidfile.rb +58 -0
  38. data/lib/solid_queue/processes/poller.rb +24 -0
  39. data/lib/solid_queue/processes/procline.rb +11 -0
  40. data/lib/solid_queue/processes/registrable.rb +69 -0
  41. data/lib/solid_queue/processes/runnable.rb +77 -0
  42. data/lib/solid_queue/processes/signals.rb +69 -0
  43. data/lib/solid_queue/processes/supervised.rb +38 -0
  44. data/lib/solid_queue/supervisor.rb +182 -0
  45. data/lib/solid_queue/tasks.rb +16 -0
  46. data/lib/solid_queue/version.rb +3 -0
  47. data/lib/solid_queue/worker.rb +54 -0
  48. data/lib/solid_queue.rb +52 -0
  49. metadata +134 -0
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ module AppExecutor
5
+ def wrap_in_app_executor(&block)
6
+ if SolidQueue.app_executor
7
+ SolidQueue.app_executor.wrap(&block)
8
+ else
9
+ yield
10
+ end
11
+ end
12
+
13
+ def handle_thread_error(error)
14
+ SolidQueue.logger.error("[SolidQueue] #{error}")
15
+
16
+ if SolidQueue.on_thread_error
17
+ SolidQueue.on_thread_error.call(error)
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Configuration
5
+ WORKER_DEFAULTS = {
6
+ queues: "*",
7
+ threads: 5,
8
+ processes: 1,
9
+ polling_interval: 0.1
10
+ }
11
+
12
+ DISPATCHER_DEFAULTS = {
13
+ batch_size: 500,
14
+ polling_interval: 1,
15
+ concurrency_maintenance_interval: 600
16
+ }
17
+
18
+ def initialize(mode: :work, load_from: nil)
19
+ @mode = mode
20
+ @raw_config = config_from(load_from)
21
+ end
22
+
23
+ def processes
24
+ case mode
25
+ when :dispatch then dispatchers
26
+ when :work then workers
27
+ when :all then dispatchers + workers
28
+ else raise "Invalid mode #{mode}"
29
+ end
30
+ end
31
+
32
+ def workers
33
+ if mode.in? %i[ work all]
34
+ workers_options.flat_map do |worker_options|
35
+ processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
36
+ processes.times.collect { SolidQueue::Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
37
+ end
38
+ else
39
+ []
40
+ end
41
+ end
42
+
43
+ def dispatchers
44
+ if mode.in? %i[ dispatch all]
45
+ dispatchers_options.flat_map do |dispatcher_options|
46
+ SolidQueue::Dispatcher.new(**dispatcher_options)
47
+ end
48
+ end
49
+ end
50
+
51
+ def max_number_of_threads
52
+ # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
53
+ workers_options.map { |options| options[:threads] }.max + 2
54
+ end
55
+
56
+ private
57
+ attr_reader :raw_config, :mode
58
+
59
+ DEFAULT_CONFIG_FILE_PATH = "config/solid_queue.yml"
60
+
61
+ def config_from(file_or_hash, env: Rails.env)
62
+ config = load_config_from(file_or_hash)
63
+ config[env.to_sym] ? config[env.to_sym] : config
64
+ end
65
+
66
+ def workers_options
67
+ @workers_options ||= (raw_config[:workers] || [ WORKER_DEFAULTS ])
68
+ .map { |options| options.dup.symbolize_keys }
69
+ end
70
+
71
+ def dispatchers_options
72
+ @dispatchers_options ||= (raw_config[:dispatchers] || [ DISPATCHER_DEFAULTS ])
73
+ .map { |options| options.dup.symbolize_keys }
74
+ end
75
+
76
+ def load_config_from(file_or_hash)
77
+ case file_or_hash
78
+ when Pathname then load_config_file file_or_hash
79
+ when String then load_config_file Pathname.new(file_or_hash)
80
+ when NilClass then load_config_file default_config_file
81
+ when Hash then file_or_hash.dup
82
+ else raise "Solid Queue cannot be initialized with #{file_or_hash.inspect}"
83
+ end
84
+ end
85
+
86
+ def load_config_file(file)
87
+ if file.exist?
88
+ ActiveSupport::ConfigurationFile.parse(file).deep_symbolize_keys
89
+ else
90
+ raise "Configuration file not found in #{file}"
91
+ end
92
+ end
93
+
94
+ def default_config_file
95
+ path_to_file = ENV["SOLID_QUEUE_CONFIG"] || DEFAULT_CONFIG_FILE_PATH
96
+
97
+ Rails.root.join(path_to_file).tap do |config_file|
98
+ raise "Configuration for Solid Queue not found in #{config_file}" unless config_file.exist?
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Dispatcher < Processes::Base
5
+ include Processes::Runnable, Processes::Poller
6
+
7
+ attr_accessor :batch_size, :concurrency_maintenance_interval
8
+
9
+ set_callback :boot, :after, :launch_concurrency_maintenance
10
+ set_callback :shutdown, :before, :stop_concurrency_maintenance
11
+
12
+ def initialize(**options)
13
+ options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
14
+
15
+ @batch_size = options[:batch_size]
16
+ @polling_interval = options[:polling_interval]
17
+ @concurrency_maintenance_interval = options[:concurrency_maintenance_interval]
18
+ end
19
+
20
+ private
21
+ def run
22
+ batch = dispatch_next_batch
23
+
24
+ unless batch.size > 0
25
+ procline "waiting"
26
+ interruptible_sleep(polling_interval)
27
+ end
28
+ end
29
+
30
+ def dispatch_next_batch
31
+ with_polling_volume do
32
+ SolidQueue::ScheduledExecution.dispatch_next_batch(batch_size)
33
+ end
34
+ end
35
+
36
+ def launch_concurrency_maintenance
37
+ @concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: concurrency_maintenance_interval) do
38
+ expire_semaphores
39
+ unblock_blocked_executions
40
+ end
41
+
42
+ @concurrency_maintenance_task.add_observer do |_, _, error|
43
+ handle_thread_error(error) if error
44
+ end
45
+
46
+ @concurrency_maintenance_task.execute
47
+ end
48
+
49
+ def stop_concurrency_maintenance
50
+ @concurrency_maintenance_task.shutdown
51
+ end
52
+
53
+ def expire_semaphores
54
+ wrap_in_app_executor do
55
+ Semaphore.expired.in_batches(of: batch_size, &:delete_all)
56
+ end
57
+ end
58
+
59
+ def unblock_blocked_executions
60
+ wrap_in_app_executor do
61
+ BlockedExecution.unblock(batch_size)
62
+ end
63
+ end
64
+
65
+ def initial_jitter
66
+ Kernel.rand(0...polling_interval)
67
+ end
68
+
69
+ def metadata
70
+ super.merge(batch_size: batch_size)
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Engine < ::Rails::Engine
5
+ isolate_namespace SolidQueue
6
+
7
+ rake_tasks do
8
+ load "solid_queue/tasks.rb"
9
+ end
10
+
11
+ config.solid_queue = ActiveSupport::OrderedOptions.new
12
+
13
+ initializer "solid_queue.config" do
14
+ config.solid_queue.each do |name, value|
15
+ SolidQueue.public_send("#{name}=", value)
16
+ end
17
+ end
18
+
19
+ initializer "solid_queue.app_executor", before: :run_prepare_callbacks do |app|
20
+ config.solid_queue.app_executor ||= app.executor
21
+ config.solid_queue.on_thread_error ||= -> (exception) { Rails.error.report(exception, handled: false) }
22
+
23
+ SolidQueue.app_executor = config.solid_queue.app_executor
24
+ SolidQueue.on_thread_error = config.solid_queue.on_thread_error
25
+ end
26
+
27
+ initializer "solid_queue.logger" do |app|
28
+ ActiveSupport.on_load(:solid_queue) do
29
+ self.logger = app.logger
30
+ end
31
+ end
32
+
33
+ initializer "solid_queue.active_job.extensions" do
34
+ ActiveSupport.on_load :active_job do
35
+ include ActiveJob::ConcurrencyControls
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ class Pool
5
+ include AppExecutor
6
+
7
+ attr_reader :size
8
+
9
+ delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor
10
+
11
+ def initialize(size, on_idle: nil)
12
+ @size = size
13
+ @on_idle = on_idle
14
+ @available_threads = Concurrent::AtomicFixnum.new(size)
15
+ @mutex = Mutex.new
16
+ end
17
+
18
+ def post(execution)
19
+ available_threads.decrement
20
+
21
+ future = Concurrent::Future.new(args: [ execution ], executor: executor) do |thread_execution|
22
+ wrap_in_app_executor do
23
+ thread_execution.perform
24
+ ensure
25
+ available_threads.increment
26
+ mutex.synchronize { on_idle.try(:call) if idle? }
27
+ end
28
+ end
29
+
30
+ future.add_observer do |_, _, error|
31
+ handle_thread_error(error) if error
32
+ end
33
+
34
+ future.execute
35
+ end
36
+
37
+ def idle_threads
38
+ available_threads.value
39
+ end
40
+
41
+ def idle?
42
+ idle_threads > 0
43
+ end
44
+
45
+ private
46
+ attr_reader :available_threads, :on_idle, :mutex
47
+
48
+ DEFAULT_OPTIONS = {
49
+ min_threads: 0,
50
+ idletime: 60,
51
+ fallback_policy: :abort
52
+ }
53
+
54
+ def executor
55
+ @executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size)
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue
4
+ module Processes
5
+ class Base
6
+ include ActiveSupport::Callbacks
7
+ define_callbacks :boot, :shutdown
8
+
9
+ include AppExecutor, Registrable, Interruptible, Procline
10
+
11
+ private
12
+ def observe_initial_delay
13
+ interruptible_sleep(initial_jitter)
14
+ end
15
+
16
+ def boot
17
+ end
18
+
19
+ def shutdown
20
+ end
21
+
22
+ def initial_jitter
23
+ 0
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue::Processes
4
+ module Interruptible
5
+ def wake_up
6
+ interrupt
7
+ end
8
+
9
+ private
10
+ SELF_PIPE_BLOCK_SIZE = 11
11
+
12
+ def interrupt
13
+ self_pipe[:writer].write_nonblock( "." )
14
+ rescue Errno::EAGAIN, Errno::EINTR
15
+ # Ignore writes that would block and retry
16
+ # if another signal arrived while writing
17
+ retry
18
+ end
19
+
20
+ def interruptible_sleep(time)
21
+ if time > 0 && self_pipe[:reader].wait_readable(time)
22
+ loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) }
23
+ end
24
+ rescue Errno::EAGAIN, Errno::EINTR
25
+ end
26
+
27
+ # Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html)
28
+ def self_pipe
29
+ @self_pipe ||= create_self_pipe
30
+ end
31
+
32
+ def create_self_pipe
33
+ reader, writer = IO.pipe
34
+ { reader: reader, writer: writer }
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue::Processes
4
+ class Pidfile
5
+ def initialize(path)
6
+ @path = path
7
+ @pid = ::Process.pid
8
+ end
9
+
10
+ def setup
11
+ check_status
12
+ write_file
13
+ set_at_exit_hook
14
+ end
15
+
16
+ def delete
17
+ delete_file
18
+ end
19
+
20
+ private
21
+ attr_reader :path, :pid
22
+
23
+ def check_status
24
+ if ::File.exist?(path)
25
+ existing_pid = ::File.open(path).read.strip.to_i
26
+ existing_pid > 0 && ::Process.kill(0, existing_pid)
27
+
28
+ already_running!
29
+ else
30
+ FileUtils.mkdir_p File.dirname(path)
31
+ end
32
+ rescue Errno::ESRCH => e
33
+ # Process is dead, ignore, just delete the file
34
+ delete
35
+ rescue Errno::EPERM
36
+ already_running!
37
+ end
38
+
39
+ def write_file
40
+ ::File.open(path, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |file| file.write(pid.to_s) }
41
+ rescue Errno::EEXIST
42
+ check_status
43
+ retry
44
+ end
45
+
46
+ def set_at_exit_hook
47
+ at_exit { delete if ::Process.pid == pid }
48
+ end
49
+
50
+ def delete_file
51
+ ::File.delete(path) if ::File.exist?(path)
52
+ end
53
+
54
+ def already_running!
55
+ abort "A Solid Queue supervisor is already running. Check #{path}"
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue::Processes
4
+ module Poller
5
+ extend ActiveSupport::Concern
6
+
7
+ included do
8
+ attr_accessor :polling_interval
9
+ end
10
+
11
+ private
12
+ def with_polling_volume
13
+ if SolidQueue.silence_polling?
14
+ ActiveRecord::Base.logger.silence { yield }
15
+ else
16
+ yield
17
+ end
18
+ end
19
+
20
+ def metadata
21
+ super.merge(polling_interval: polling_interval)
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue::Processes
4
+ module Procline
5
+ # Sets the procline ($0)
6
+ # solid-queue-supervisor(0.1.0): <string>
7
+ def procline(string)
8
+ $0 = "solid-queue-#{self.class.name.demodulize.downcase}(#{SolidQueue::VERSION}): #{string}"
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue::Processes
4
+ module Registrable
5
+ extend ActiveSupport::Concern
6
+
7
+ included do
8
+ set_callback :boot, :after, :register
9
+ set_callback :boot, :after, :launch_heartbeat
10
+
11
+ set_callback :shutdown, :before, :stop_heartbeat
12
+ set_callback :shutdown, :after, :deregister
13
+ end
14
+
15
+ def inspect
16
+ "#{kind}(pid=#{process_pid}, hostname=#{hostname}, metadata=#{metadata})"
17
+ end
18
+ alias to_s inspect
19
+
20
+ private
21
+ attr_accessor :process
22
+
23
+ def register
24
+ @process = SolidQueue::Process.register \
25
+ kind: self.class.name.demodulize,
26
+ pid: process_pid,
27
+ hostname: hostname,
28
+ supervisor: try(:supervisor),
29
+ metadata: metadata
30
+ end
31
+
32
+ def deregister
33
+ process.deregister if registered?
34
+ end
35
+
36
+ def registered?
37
+ process&.persisted?
38
+ end
39
+
40
+ def launch_heartbeat
41
+ @heartbeat_task = Concurrent::TimerTask.new(execution_interval: SolidQueue.process_heartbeat_interval) { heartbeat }
42
+ @heartbeat_task.execute
43
+ end
44
+
45
+ def stop_heartbeat
46
+ @heartbeat_task&.shutdown
47
+ end
48
+
49
+ def heartbeat
50
+ process.heartbeat
51
+ end
52
+
53
+ def kind
54
+ self.class.name.demodulize
55
+ end
56
+
57
+ def hostname
58
+ @hostname ||= Socket.gethostname
59
+ end
60
+
61
+ def process_pid
62
+ @pid ||= ::Process.pid
63
+ end
64
+
65
+ def metadata
66
+ {}
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue::Processes
4
+ module Runnable
5
+ include Supervised
6
+
7
+ def start
8
+ @stopping = false
9
+
10
+ observe_initial_delay
11
+ run_callbacks(:boot) { boot }
12
+
13
+ start_loop
14
+ end
15
+
16
+ def stop
17
+ @stopping = true
18
+ @thread&.join
19
+ end
20
+
21
+ private
22
+ attr_writer :mode
23
+
24
+ DEFAULT_MODE = :async
25
+
26
+ def mode
27
+ (@mode || DEFAULT_MODE).to_s.inquiry
28
+ end
29
+
30
+ def boot
31
+ register_signal_handlers if supervised?
32
+ SolidQueue.logger.info("[SolidQueue] Starting #{self}")
33
+ end
34
+
35
+ def start_loop
36
+ if mode.async?
37
+ @thread = Thread.new { do_start_loop }
38
+ else
39
+ do_start_loop
40
+ end
41
+ end
42
+
43
+ def do_start_loop
44
+ loop do
45
+ break if shutting_down?
46
+
47
+ run
48
+ end
49
+ ensure
50
+ run_callbacks(:shutdown) { shutdown }
51
+ end
52
+
53
+ def shutting_down?
54
+ stopping? || supervisor_went_away? || finished?
55
+ end
56
+
57
+ def run
58
+ raise NotImplementedError
59
+ end
60
+
61
+ def stopping?
62
+ @stopping
63
+ end
64
+
65
+ def finished?
66
+ running_inline? && all_work_completed?
67
+ end
68
+
69
+ def all_work_completed?
70
+ false
71
+ end
72
+
73
+ def running_inline?
74
+ mode.inline?
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueue::Processes
4
+ class GracefulTerminationRequested < Interrupt; end
5
+ class ImmediateTerminationRequested < Interrupt; end
6
+
7
+ module Signals
8
+ extend ActiveSupport::Concern
9
+
10
+ private
11
+ SIGNALS = %i[ QUIT INT TERM ]
12
+
13
+ def register_signal_handlers
14
+ SIGNALS.each do |signal|
15
+ trap(signal) do
16
+ signal_queue << signal
17
+ interrupt
18
+ end
19
+ end
20
+ end
21
+
22
+ def restore_default_signal_handlers
23
+ SIGNALS.each do |signal|
24
+ trap(signal, :DEFAULT)
25
+ end
26
+ end
27
+
28
+ def process_signal_queue
29
+ while signal = signal_queue.shift
30
+ handle_signal(signal)
31
+ end
32
+ end
33
+
34
+ def handle_signal(signal)
35
+ case signal
36
+ when :TERM, :INT
37
+ request_graceful_termination
38
+ when :QUIT
39
+ request_immediate_termination
40
+ else
41
+ SolidQueue.logger.warn "Received unhandled signal #{signal}"
42
+ end
43
+ end
44
+
45
+ def request_graceful_termination
46
+ raise GracefulTerminationRequested
47
+ end
48
+
49
+ def request_immediate_termination
50
+ raise ImmediateTerminationRequested
51
+ end
52
+
53
+ def signal_processes(pids, signal)
54
+ pids.each do |pid|
55
+ signal_process pid, signal
56
+ end
57
+ end
58
+
59
+ def signal_process(pid, signal)
60
+ ::Process.kill signal, pid
61
+ rescue Errno::ESRCH
62
+ # Ignore, process died before
63
+ end
64
+
65
+ def signal_queue
66
+ @signal_queue ||= []
67
+ end
68
+ end
69
+ end