solid_queue 0.1.1

data/app/models/solid_queue/job.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class SolidQueue::Job < SolidQueue::Record
+  include Executable
+
+  if Gem::Version.new(Rails.version) >= Gem::Version.new("7.1")
+    serialize :arguments, coder: JSON
+  else
+    serialize :arguments, JSON
+  end
+
+  DEFAULT_PRIORITY = 0
+  DEFAULT_QUEUE_NAME = "default"
+
+  class << self
+    def enqueue_active_job(active_job, scheduled_at: Time.current)
+      enqueue \
+        queue_name: active_job.queue_name,
+        active_job_id: active_job.job_id,
+        priority: active_job.priority,
+        scheduled_at: scheduled_at,
+        class_name: active_job.class.name,
+        arguments: active_job.serialize,
+        concurrency_key: active_job.try(:concurrency_key)
+    end
+
+    def enqueue(**kwargs)
+      create!(**kwargs.compact.with_defaults(defaults)).tap do
+        SolidQueue.logger.debug "[SolidQueue] Enqueued job #{kwargs}"
+      end
+    end
+
+    private
+      def defaults
+        { queue_name: DEFAULT_QUEUE_NAME, priority: DEFAULT_PRIORITY }
+      end
+  end
+end

data/app/models/solid_queue/pause.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Pause < Record
+  end
+end

data/app/models/solid_queue/process/prunable.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SolidQueue::Process::Prunable
+  extend ActiveSupport::Concern
+
+  included do
+    scope :prunable, -> { where("last_heartbeat_at <= ?", SolidQueue.process_alive_threshold.ago) }
+  end
+
+  class_methods do
+    def prune
+      prunable.lock.find_in_batches(batch_size: 50) do |batch|
+        batch.each do |process|
+          SolidQueue.logger.info("[SolidQueue] Pruning dead process #{process.id} - #{process.metadata}")
+          process.deregister
+        end
+      end
+    end
+  end
+end

data/app/models/solid_queue/process.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class SolidQueue::Process < SolidQueue::Record
+  include Prunable
+
+  belongs_to :supervisor, class_name: "SolidQueue::Process", optional: true, inverse_of: :forks
+  has_many :forks, class_name: "SolidQueue::Process", inverse_of: :supervisor, foreign_key: :supervisor_id, dependent: :destroy
+  has_many :claimed_executions
+
+  store :metadata, coder: JSON
+
+  after_destroy -> { claimed_executions.release_all }
+
+  def self.register(**attributes)
+    create!(attributes.merge(last_heartbeat_at: Time.current))
+  end
+
+  def heartbeat
+    touch(:last_heartbeat_at)
+  end
+
+  def deregister
+    destroy!
+  rescue Exception
+    SolidQueue.logger.error("[SolidQueue] Error deregistering process #{id} - #{metadata}")
+    raise
+  end
+end

data/app/models/solid_queue/queue.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Queue
+    attr_accessor :name
+
+    class << self
+      def all
+        Job.select(:queue_name).distinct.collect do |job|
+          new(job.queue_name)
+        end
+      end
+
+      def find_by_name(name)
+        new(name)
+      end
+    end
+
+    def initialize(name)
+      @name = name
+    end
+
+    def paused?
+      Pause.exists?(queue_name: name)
+    end
+
+    def pause
+      Pause.create_or_find_by!(queue_name: name)
+    end
+
+    def resume
+      Pause.where(queue_name: name).delete_all
+    end
+
+    def clear
+      Job.where(queue_name: name).each(&:discard)
+    end
+
+    def size
+      @size ||= ReadyExecution.queued_as(name).count
+    end
+
+    def ==(queue)
+      name == queue.name
+    end
+    alias_method :eql?, :==
+
+    def hash
+      name.hash
+    end
+  end
+end

data/app/models/solid_queue/queue_selector.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class QueueSelector
+    attr_reader :raw_queues, :relation
+
+    def initialize(queue_list, relation)
+      @raw_queues = Array(queue_list).map { |queue| queue.to_s.strip }.presence || [ "*" ]
+      @relation = relation
+    end
+
+    def scoped_relations
+      case
+      when all?  then [ relation.all ]
+      when none? then [ relation.none ]
+      else
+        queue_names.map { |queue_name| relation.queued_as(queue_name) }
+      end
+    end
+
+    private
+      def all?
+        include_all_queues? && paused_queues.empty?
+      end
+
+      def none?
+        queue_names.empty?
+      end
+
+      def queue_names
+        @queue_names ||= eligible_queues - paused_queues
+      end
+
+      def eligible_queues
+        if include_all_queues? then all_queues
+        else
+          exact_names + prefixed_names
+        end
+      end
+
+      def include_all_queues?
+        "*".in? raw_queues
+      end
+
+      def exact_names
+        raw_queues.select { |queue| !queue.include?("*") }
+      end
+
+      def prefixed_names
+        if prefixes.empty? then []
+        else
+          relation.where(([ "queue_name LIKE ?" ] * prefixes.count).join(" OR "), *prefixes).distinct(:queue_name).pluck(:queue_name)
+        end
+      end
+
+      def prefixes
+        @prefixes ||= raw_queues.select { |queue| queue.ends_with?("*") }.map { |queue| queue.tr("*", "%") }
+      end
+
+      def all_queues
+        relation.distinct(:queue_name).pluck(:queue_name)
+      end
+
+      def paused_queues
+        @paused_queues ||= Pause.all.pluck(:queue_name)
+      end
+  end
+end

data/app/models/solid_queue/ready_execution.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class ReadyExecution < Execution
+    scope :queued_as, ->(queue_name) { where(queue_name: queue_name) }
+
+    assume_attributes_from_job
+
+    class << self
+      def claim(queue_list, limit, process_id)
+        QueueSelector.new(queue_list, self).scoped_relations.flat_map do |queue_relation|
+          select_and_lock(queue_relation, process_id, limit).tap do |locked|
+            limit -= locked.size
+          end
+        end
+      end
+
+      private
+        def select_and_lock(queue_relation, process_id, limit)
+          return [] if limit <= 0
+
+          transaction do
+            job_ids = select_candidates(queue_relation, limit)
+            lock_candidates(job_ids, process_id)
+          end
+        end
+
+        def select_candidates(queue_relation, limit)
+          queue_relation.ordered.limit(limit).lock.pluck(:job_id)
+        end
+
+        def lock_candidates(job_ids, process_id)
+          return [] if job_ids.none?
+
+          SolidQueue::ClaimedExecution.claiming(job_ids, process_id) do |claimed|
+            where(job_id: claimed.pluck(:job_id)).delete_all
+          end
+        end
+    end
+  end
+end

data/app/models/solid_queue/record.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Record < ActiveRecord::Base
+    self.abstract_class = true
+
+    connects_to **SolidQueue.connects_to if SolidQueue.connects_to
+
+    def self.lock(...)
+      if SolidQueue.use_skip_locked
+        super(Arel.sql("FOR UPDATE SKIP LOCKED"))
+      else
+        super
+      end
+    end
+  end
+end
+
+ActiveSupport.run_load_hooks :solid_queue_record, SolidQueue::Record

data/app/models/solid_queue/scheduled_execution.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class ScheduledExecution < Execution
+    scope :due, -> { where(scheduled_at: ..Time.current) }
+    scope :ordered, -> { order(scheduled_at: :asc, priority: :asc) }
+    scope :next_batch, ->(batch_size) { due.ordered.limit(batch_size) }
+
+    assume_attributes_from_job :scheduled_at
+
+    class << self
+      def dispatch_next_batch(batch_size)
+        transaction do
+          job_ids = next_batch(batch_size).lock.pluck(:job_id)
+          if job_ids.empty? then []
+          else
+            dispatch_batch(job_ids)
+          end
+        end
+      end
+
+      private
+        def dispatch_batch(job_ids)
+          jobs = Job.where(id: job_ids)
+          with_concurrency_limits, without_concurrency_limits = jobs.partition(&:concurrency_limited?)
+
+          dispatch_at_once(without_concurrency_limits)
+          dispatch_one_by_one(with_concurrency_limits)
+
+          successfully_dispatched(job_ids).tap do |dispatched_job_ids|
+            where(job_id: dispatched_job_ids).delete_all
+            SolidQueue.logger.info("[SolidQueue] Dispatched scheduled batch with #{dispatched_job_ids.size} jobs")
+          end
+        end
+
+        def dispatch_at_once(jobs)
+          ReadyExecution.insert_all ready_rows_from_batch(jobs)
+        end
+
+        def dispatch_one_by_one(jobs)
+          jobs.each(&:dispatch)
+        end
+
+        def ready_rows_from_batch(jobs)
+          prepared_at = Time.current
+
+          jobs.map do |job|
+            { job_id: job.id, queue_name: job.queue_name, priority: job.priority, created_at: prepared_at }
+          end
+        end
+
+        def successfully_dispatched(job_ids)
+          dispatched_and_ready(job_ids) + dispatched_and_blocked(job_ids)
+        end
+
+        def dispatched_and_ready(job_ids)
+          ReadyExecution.where(job_id: job_ids).pluck(:job_id)
+        end
+
+        def dispatched_and_blocked(job_ids)
+          BlockedExecution.where(job_id: job_ids).pluck(:job_id)
+        end
+    end
+  end
+end

data/app/models/solid_queue/semaphore.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class SolidQueue::Semaphore < SolidQueue::Record
+  scope :available, -> { where("value > 0") }
+  scope :expired, -> { where(expires_at: ...Time.current) }
+
+  class << self
+    def wait(job)
+      Proxy.new(job, self).wait
+    end
+
+    def signal(job)
+      Proxy.new(job, self).signal
+    end
+  end
+
+  class Proxy
+    def initialize(job, proxied_class)
+      @job = job
+      @proxied_class = proxied_class
+    end
+
+    def wait
+      if semaphore = proxied_class.find_by(key: key)
+        semaphore.value > 0 && attempt_decrement
+      else
+        attempt_creation
+      end
+    end
+
+    def signal
+      attempt_increment
+    end
+
+    private
+      attr_reader :job, :proxied_class
+
+      def attempt_creation
+        proxied_class.create!(key: key, value: limit - 1, expires_at: expires_at)
+        true
+      rescue ActiveRecord::RecordNotUnique
+        attempt_decrement
+      end
+
+      def attempt_decrement
+        proxied_class.available.where(key: key).update_all([ "value = value - 1, expires_at = ?", expires_at ]) > 0
+      end
+
+      def attempt_increment
+        proxied_class.where(key: key, value: ...limit).update_all([ "value = value + 1, expires_at = ?", expires_at ]) > 0
+      end
+
+      def key
+        job.concurrency_key
+      end
+
+      def expires_at
+        job.concurrency_duration.from_now
+      end
+
+      def limit
+        job.concurrency_limit
+      end
+  end
+end

data/config/routes.rb

@@ -0,0 +1,2 @@
+Rails.application.routes.draw do
+end

data/db/migrate/20231211200639_create_solid_queue_tables.rb

@@ -0,0 +1,100 @@
+class CreateSolidQueueTables < ActiveRecord::Migration[7.1]
+  def change
+    create_table :solid_queue_jobs do |t|
+      t.string :queue_name, null: false
+      t.string :class_name, null: false, index: true
+      t.text :arguments
+      t.integer :priority, default: 0, null: false
+      t.string :active_job_id, index: true
+      t.datetime :scheduled_at
+      t.datetime :finished_at, index: true
+      t.string :concurrency_key
+
+      t.timestamps
+
+      t.index [ :queue_name, :finished_at ], name: "index_solid_queue_jobs_for_filtering"
+      t.index [ :scheduled_at, :finished_at ], name: "index_solid_queue_jobs_for_alerting"
+    end
+
+    create_table :solid_queue_scheduled_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.datetime :scheduled_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :scheduled_at, :priority, :job_id ], name: "index_solid_queue_dispatch_all"
+    end
+
+    create_table :solid_queue_ready_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :priority, :job_id ], name: "index_solid_queue_poll_all"
+      t.index [ :queue_name, :priority, :job_id ], name: "index_solid_queue_poll_by_queue"
+    end
+
+    create_table :solid_queue_claimed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.bigint :process_id
+      t.datetime :created_at, null: false
+
+      t.index [ :process_id, :job_id ]
+    end
+
+    create_table :solid_queue_blocked_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.string :concurrency_key, null: false
+      t.datetime :expires_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :expires_at, :concurrency_key ], name: "index_solid_queue_blocked_executions_for_maintenance"
+    end
+
+    create_table :solid_queue_failed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.text :error
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_pauses do |t|
+      t.string :queue_name, null: false, index: { unique: true }
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_processes do |t|
+      t.string :kind, null: false
+      t.datetime :last_heartbeat_at, null: false, index: true
+      t.bigint :supervisor_id, index: true
+
+      t.integer :pid, null: false
+      t.string :hostname
+      t.text :metadata
+
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_semaphores do |t|
+      t.string :key, null: false, index: { unique: true }
+      t.integer :value, default: 1, null: false
+      t.datetime :expires_at, null: false, index: true
+
+      t.timestamps
+
+      t.index [ :key, :value ], name: "index_solid_queue_semaphores_on_key_and_value"
+    end
+
+    add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end

data/lib/active_job/concurrency_controls.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module ConcurrencyControls
+    extend ActiveSupport::Concern
+
+    DEFAULT_CONCURRENCY_GROUP = ->(*) { self.class.name }
+
+    included do
+      class_attribute :concurrency_key, instance_accessor: false
+      class_attribute :concurrency_group, default: DEFAULT_CONCURRENCY_GROUP, instance_accessor: false
+
+      class_attribute :concurrency_limit
+      class_attribute :concurrency_duration, default: SolidQueue.default_concurrency_control_period
+    end
+
+    class_methods do
+      def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period)
+        self.concurrency_key = key
+        self.concurrency_limit = to
+        self.concurrency_group = group
+        self.concurrency_duration = duration
+      end
+    end
+
+    def concurrency_key
+      if self.class.concurrency_key
+        param = compute_concurrency_parameter(self.class.concurrency_key)
+
+        case param
+        when ActiveRecord::Base
+          [ concurrency_group, param.class.name, param.id ]
+        else
+          [ concurrency_group, param ]
+        end.compact.join("/")
+      end
+    end
+
+    private
+      def concurrency_group
+        compute_concurrency_parameter(self.class.concurrency_group)
+      end
+
+      def compute_concurrency_parameter(option)
+        case option
+        when String, Symbol
+          option.to_s
+        when Proc
+          instance_exec(*arguments, &option)
+        end
+      end
+  end
+end

data/lib/active_job/queue_adapters/solid_queue_adapter.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    # == Active Job SolidQueue adapter
+    #
+    # To use it set the queue_adapter config to +:solid_queue+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :solid_queue
+    class SolidQueueAdapter
+      def enqueue(active_job) # :nodoc:
+        SolidQueue::Job.enqueue_active_job(active_job).tap do |job|
+          active_job.provider_job_id = job.id
+        end
+      end
+
+      def enqueue_at(active_job, timestamp) # :nodoc:
+        SolidQueue::Job.enqueue_active_job(active_job, scheduled_at: Time.at(timestamp)).tap do |job|
+          active_job.provider_job_id = job.id
+        end
+      end
+    end
+  end
+end

data/lib/generators/solid_queue/install/USAGE

@@ -0,0 +1,9 @@
+Description:
+    Installs solid_queue as the Active Job's queue adapter
+
+Example:
+    bin/rails generate solid_queue:install
+
+    This will create:
+        Installs solid_queue migrations
+        Replaces Active Job's adapter in envionment configuration

data/lib/generators/solid_queue/install/install_generator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class SolidQueue::InstallGenerator < Rails::Generators::Base
+  class_option :skip_migrations, type: :boolean, default: nil, desc: "Skip migrations"
+
+  def add_solid_queue
+    %w[ development test production ].each do |env_name|
+      if (env_config = Pathname(destination_root).join("config/environments/#{env_name}.rb")).exist?
+        gsub_file env_config, /(# )?config\.active_job\.queue_adapter\s+=.*/, "config.active_job.queue_adapter = :solid_queue"
+      end
+    end
+  end
+
+  def create_migrations
+    unless options[:skip_migrations]
+      rails_command "railties:install:migrations FROM=solid_queue", inline: true
+    end
+  end
+end

data/lib/puma/plugin/solid_queue.rb

@@ -0,0 +1,63 @@
+require "puma/plugin"
+
+Puma::Plugin.create do
+  attr_reader :puma_pid, :solid_queue_pid, :log_writer
+
+  def start(launcher)
+    @log_writer = launcher.log_writer
+    @puma_pid = $$
+    @solid_queue_pid = fork do
+      Thread.new { monitor_puma }
+      SolidQueue::Supervisor.start(mode: :all)
+    end
+
+    launcher.events.on_stopped { stop_solid_queue }
+
+    in_background do
+      monitor_solid_queue
+    end
+  end
+
+  private
+    def stop_solid_queue
+      Process.waitpid(solid_queue_pid, Process::WNOHANG)
+      log "Stopping Solid Queue..."
+      Process.kill(:INT, solid_queue_pid) if solid_queue_pid
+      Process.wait(solid_queue_pid)
+    rescue Errno::ECHILD, Errno::ESRCH
+    end
+
+    def monitor_puma
+      monitor(:puma_dead?, "Detected Puma has gone away, stopping Solid Queue...")
+    end
+
+    def monitor_solid_queue
+      monitor(:solid_queue_dead?, "Detected Solid Queue has gone away, stopping Puma...")
+    end
+
+    def monitor(process_dead, message)
+      loop do
+        if send(process_dead)
+          log message
+          Process.kill(:INT, $$)
+          break
+        end
+        sleep 2
+      end
+    end
+
+    def solid_queue_dead?
+      Process.waitpid(solid_queue_pid, Process::WNOHANG)
+      false
+    rescue Errno::ECHILD, Errno::ESRCH
+      true
+    end
+
+    def puma_dead?
+      Process.ppid != puma_pid
+    end
+
+    def log(...)
+      log_writer.log(...)
+    end
+end