solid_queue 0.1.1

data/lib/solid_queue/processes/supervised.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Supervised
+    extend ActiveSupport::Concern
+
+    included do
+      attr_reader :supervisor
+    end
+
+    def supervised_by(process)
+      self.mode = :supervised
+      @supervisor = process
+    end
+
+    private
+      def supervisor_went_away?
+        supervised? && supervisor&.pid != ::Process.ppid
+      end
+
+      def supervised?
+        mode.supervised?
+      end
+
+      def register_signal_handlers
+        %w[ INT TERM ].each do |signal|
+          trap(signal) do
+            stop
+            interrupt
+          end
+        end
+
+        trap(:QUIT) do
+          exit!
+        end
+      end
+  end
+end

data/lib/solid_queue/supervisor.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor < Processes::Base
+    include Processes::Signals
+
+    set_callback :boot, :after, :launch_process_prune
+
+    class << self
+      def start(mode: :work, load_configuration_from: nil)
+        SolidQueue.supervisor = true
+        configuration = Configuration.new(mode: mode, load_from: load_configuration_from)
+
+        new(*configuration.processes).start
+      end
+    end
+
+    def initialize(*configured_processes)
+      @configured_processes = Array(configured_processes)
+      @forks = {}
+    end
+
+    def start
+      run_callbacks(:boot) { boot }
+
+      supervise
+    rescue Processes::GracefulTerminationRequested
+      graceful_termination
+    rescue Processes::ImmediateTerminationRequested
+      immediate_termination
+    ensure
+      run_callbacks(:shutdown) { shutdown }
+    end
+
+    private
+      attr_reader :configured_processes, :forks
+
+      def boot
+        sync_std_streams
+        setup_pidfile
+        register_signal_handlers
+      end
+
+      def supervise
+        start_forks
+
+        loop do
+          procline "supervising #{forks.keys.join(", ")}"
+
+          process_signal_queue
+          reap_and_replace_terminated_forks
+          interruptible_sleep(1.second)
+        end
+      end
+
+      def sync_std_streams
+        STDOUT.sync = STDERR.sync = true
+      end
+
+      def setup_pidfile
+        @pidfile = if SolidQueue.supervisor_pidfile
+          Processes::Pidfile.new(SolidQueue.supervisor_pidfile).tap(&:setup)
+        end
+      end
+
+      def launch_process_prune
+        @prune_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) { prune_dead_processes }
+        @prune_task.execute
+      end
+
+      def start_forks
+        configured_processes.each { |configured_process| start_fork(configured_process) }
+      end
+
+      def shutdown
+        stop_process_prune
+        restore_default_signal_handlers
+        delete_pidfile
+      end
+
+      def graceful_termination
+        procline "terminating gracefully"
+        term_forks
+
+        wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
+          reap_terminated_forks
+        end
+
+        immediate_termination unless all_forks_terminated?
+      end
+
+      def immediate_termination
+        procline "terminating immediately"
+        quit_forks
+      end
+
+      def term_forks
+        signal_processes(forks.keys, :TERM)
+      end
+
+      def quit_forks
+        signal_processes(forks.keys, :QUIT)
+      end
+
+      def stop_process_prune
+        @prune_task&.shutdown
+      end
+
+      def delete_pidfile
+        @pidfile&.delete
+      end
+
+      def prune_dead_processes
+        wrap_in_app_executor do
+          SolidQueue::Process.prune
+        end
+      end
+
+      def start_fork(configured_process)
+        configured_process.supervised_by process
+
+        pid = fork do
+          configured_process.start
+        end
+
+        forks[pid] = configured_process
+      end
+
+      def reap_and_replace_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          replace_fork(pid, status)
+        end
+      end
+
+      def reap_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          forks.delete(pid)
+        end
+      rescue SystemCallError
+        # All children already reaped
+      end
+
+      def replace_fork(pid, status)
+        if supervised_fork = forks.delete(pid)
+          SolidQueue.logger.info "[SolidQueue] Restarting fork[#{status.pid}] (status: #{status.exitstatus})"
+          start_fork(supervised_fork)
+        else
+          SolidQueue.logger.info "[SolidQueue] Tried to replace fork[#{pid}] (status: #{status.exitstatus}, fork[#{status.pid}]), but it had already died  (status: #{status.exitstatus})"
+        end
+      end
+
+      def all_forks_terminated?
+        forks.empty?
+      end
+
+      def wait_until(timeout, condition, &block)
+        if timeout > 0
+          deadline = monotonic_time_now + timeout
+
+          while monotonic_time_now < deadline && !condition.call
+            sleep 0.1
+            block.call
+          end
+        else
+          while !condition.call
+            sleep 0.5
+            block.call
+          end
+        end
+      end
+
+      def monotonic_time_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+  end
+end

data/lib/solid_queue/tasks.rb

@@ -0,0 +1,16 @@
+namespace :solid_queue do
+  desc "start solid_queue supervisor to dispatch and process jobs"
+  task start: :environment do
+    SolidQueue::Supervisor.start(mode: :all)
+  end
+
+  desc "start solid_queue supervisor to process jobs"
+  task work: :environment do
+    SolidQueue::Supervisor.start(mode: :work)
+  end
+
+  desc "start solid_queue dispatcher to enqueue scheduled jobs"
+  task dispatch: :environment do
+    SolidQueue::Supervisor.start(mode: :dispatch)
+  end
+end

data/lib/solid_queue/version.rb

@@ -0,0 +1,3 @@
+module SolidQueue
+  VERSION = "0.1.1"
+end

data/lib/solid_queue/worker.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Worker < Processes::Base
+    include Processes::Runnable, Processes::Poller
+
+    attr_accessor :queues, :pool
+
+    def initialize(**options)
+      options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
+
+      @polling_interval = options[:polling_interval]
+      @queues = Array(options[:queues])
+      @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
+    end
+
+    private
+      def run
+        polled_executions = poll
+
+        if polled_executions.size > 0
+          procline "performing #{polled_executions.count} jobs"
+
+          polled_executions.each do |execution|
+            pool.post(execution)
+          end
+        else
+          procline "waiting for jobs in #{queues.join(",")}"
+          interruptible_sleep(polling_interval)
+        end
+      end
+
+      def poll
+        with_polling_volume do
+          SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process.id)
+        end
+      end
+
+      def shutdown
+        super
+
+        pool.shutdown
+        pool.wait_for_termination(SolidQueue.shutdown_timeout)
+      end
+
+      def all_work_completed?
+        SolidQueue::ReadyExecution.queued_as(queues).empty?
+      end
+
+      def metadata
+        super.merge(queues: queues.join(","), thread_pool_size: pool.size)
+      end
+  end
+end

data/lib/solid_queue.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "solid_queue/version"
+require "solid_queue/engine"
+
+require "active_job/queue_adapters/solid_queue_adapter"
+require "active_job/concurrency_controls"
+
+require "solid_queue/app_executor"
+require "solid_queue/processes/supervised"
+require "solid_queue/processes/registrable"
+require "solid_queue/processes/interruptible"
+require "solid_queue/processes/pidfile"
+require "solid_queue/processes/procline"
+require "solid_queue/processes/poller"
+require "solid_queue/processes/base"
+require "solid_queue/processes/runnable"
+require "solid_queue/processes/signals"
+require "solid_queue/configuration"
+require "solid_queue/pool"
+require "solid_queue/worker"
+require "solid_queue/dispatcher"
+require "solid_queue/supervisor"
+
+module SolidQueue
+  mattr_accessor :logger, default: ActiveSupport::Logger.new($stdout)
+  mattr_accessor :app_executor, :on_thread_error, :connects_to
+
+  mattr_accessor :use_skip_locked, default: true
+
+  mattr_accessor :process_heartbeat_interval, default: 60.seconds
+  mattr_accessor :process_alive_threshold, default: 5.minutes
+
+  mattr_accessor :shutdown_timeout, default: 5.seconds
+
+  mattr_accessor :silence_polling, default: false
+
+  mattr_accessor :supervisor_pidfile
+  mattr_accessor :supervisor, default: false
+
+  mattr_accessor :preserve_finished_jobs, default: true
+  mattr_accessor :clear_finished_jobs_after, default: 1.day
+  mattr_accessor :default_concurrency_control_period, default: 3.minutes
+
+  def self.supervisor?
+    supervisor
+  end
+
+  def self.silence_polling?
+    silence_polling
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: solid_queue
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Rosa Gutierrez
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-12-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.3.1
+- !ruby/object:Gem::Dependency
+  name: debug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Database-backed Active Job backend.
+email:
+- rosa@37signals.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/models/solid_queue/blocked_execution.rb
+- app/models/solid_queue/claimed_execution.rb
+- app/models/solid_queue/execution.rb
+- app/models/solid_queue/execution/job_attributes.rb
+- app/models/solid_queue/failed_execution.rb
+- app/models/solid_queue/job.rb
+- app/models/solid_queue/job/clearable.rb
+- app/models/solid_queue/job/concurrency_controls.rb
+- app/models/solid_queue/job/executable.rb
+- app/models/solid_queue/pause.rb
+- app/models/solid_queue/process.rb
+- app/models/solid_queue/process/prunable.rb
+- app/models/solid_queue/queue.rb
+- app/models/solid_queue/queue_selector.rb
+- app/models/solid_queue/ready_execution.rb
+- app/models/solid_queue/record.rb
+- app/models/solid_queue/scheduled_execution.rb
+- app/models/solid_queue/semaphore.rb
+- config/routes.rb
+- db/migrate/20231211200639_create_solid_queue_tables.rb
+- lib/active_job/concurrency_controls.rb
+- lib/active_job/queue_adapters/solid_queue_adapter.rb
+- lib/generators/solid_queue/install/USAGE
+- lib/generators/solid_queue/install/install_generator.rb
+- lib/puma/plugin/solid_queue.rb
+- lib/solid_queue.rb
+- lib/solid_queue/app_executor.rb
+- lib/solid_queue/configuration.rb
+- lib/solid_queue/dispatcher.rb
+- lib/solid_queue/engine.rb
+- lib/solid_queue/pool.rb
+- lib/solid_queue/processes/base.rb
+- lib/solid_queue/processes/interruptible.rb
+- lib/solid_queue/processes/pidfile.rb
+- lib/solid_queue/processes/poller.rb
+- lib/solid_queue/processes/procline.rb
+- lib/solid_queue/processes/registrable.rb
+- lib/solid_queue/processes/runnable.rb
+- lib/solid_queue/processes/signals.rb
+- lib/solid_queue/processes/supervised.rb
+- lib/solid_queue/supervisor.rb
+- lib/solid_queue/tasks.rb
+- lib/solid_queue/version.rb
+- lib/solid_queue/worker.rb
+homepage: http://github.com/basecamp/solid_queue
+licenses:
+- MIT
+metadata:
+  homepage_uri: http://github.com/basecamp/solid_queue
+  source_code_uri: http://github.com/basecamp/solid_queue
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.20
+signing_key:
+specification_version: 4
+summary: Database-backed Active Job backend.
+test_files: []