solid_queue 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72d9dc2e30127c7b1fe101648d4d18b04af2d33491c729816e25dcf488246b54
-  data.tar.gz: e39dcad251d4124355adcedfa9cb1ee88eea12043eb8505568332d9fcacee428
+  metadata.gz: 0eb31439e2768af5f5d3fb9289ac51056570c762ad140bc54aba81ee770d5a12
+  data.tar.gz: 4d3e4c2608b0a3ed2de82c7712c8a524e8745bc4e2ddf6907de8b3723560ab04
 SHA512:
-  metadata.gz: 967732a5e679269543869ca64aa410bfc43c935ad3fb6856de0860bbe5cee58716e389b5935e48a91c6c658a84fa79aff9f5eae93ba3931344d31aff0d45c8fc
-  data.tar.gz: 523af9f1b26a137d99ade0805bdc955cb81584971308928d5e16e83d3c46861947deaaa95ff96775d7eceb913f3d8995eca14707cda4533f7c3ee39f20afb650
+  metadata.gz: 0f4271aaf7b55b86d81f97bf19bbfa2b76de1ff994a6b71bb097b89ab9585212f28c0fdbd4fbb86b74c18b49cf229e7913eaa724784b3a62f000d60c2ade3263
+  data.tar.gz: a84d91b13db4a3ec96d1afa5fb5f39abc9ea0bf4bc216c14e20f4bf7e1bf2ff414e08f78166377950351b8170cbd734aa69322feb9076a67a3f9939ebacf6573

data/MIT-LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2023 37signals
+Copyright (c) 37signals, LLC
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -31,7 +31,7 @@ $ bin/rails generate solid_queue:install
 
 This will set `solid_queue` as the Active Job's adapter in production, and will copy the required migration over to your app.
 
-Alternatively, you can add the only the migration to your app:
+Alternatively, you can add only the migration to your app:
 ```bash
 $ bin/rails solid_queue:install:migrations
 ```

data/app/models/solid_queue/claimed_execution.rb

@@ -23,6 +23,14 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
     def release_all
       includes(:job).each(&:release)
     end
+
+    def discard_all_in_batches(*)
+      raise UndiscardableError, "Can't discard jobs in progress"
+    end
+
+    def discard_all_from_jobs(*)
+      raise UndiscardableError, "Can't discard jobs in progress"
+    end
   end
 
   def perform
@@ -39,11 +47,15 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
 
   def release
     transaction do
-      job.prepare_for_execution
+      job.dispatch_bypassing_concurrency_limits
       destroy!
     end
   end
 
+  def discard
+    raise UndiscardableError, "Can't discard a job in progress"
+  end
+
   private
     def execute
       ActiveJob::Base.execute(job.arguments)

data/app/models/solid_queue/execution/dispatching.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Execution
+    module Dispatching
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def dispatch_jobs(job_ids)
+          jobs = Job.where(id: job_ids)
+
+          Job.dispatch_all(jobs).map(&:id).tap do |dispatched_job_ids|
+            where(job_id: dispatched_job_ids).delete_all
+            SolidQueue.logger.info("[SolidQueue] Dispatched #{dispatched_job_ids.size} jobs")
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/solid_queue/execution/job_attributes.rb

@@ -6,23 +6,23 @@ module SolidQueue
       extend ActiveSupport::Concern
 
       included do
-        class_attribute :assumible_attributes_from_job, instance_accessor: false, default: %i[ queue_name priority ]
+        class_attribute :assumable_attributes_from_job, instance_accessor: false, default: %i[ queue_name priority ]
       end
 
       class_methods do
         def assumes_attributes_from_job(*attribute_names)
-          self.assumible_attributes_from_job |= attribute_names
+          self.assumable_attributes_from_job |= attribute_names
           before_create -> { assume_attributes_from_job }
         end
 
         def attributes_from_job(job)
-          job.attributes.symbolize_keys.slice(*assumible_attributes_from_job)
+          job.attributes.symbolize_keys.slice(*assumable_attributes_from_job)
         end
       end
 
       private
         def assume_attributes_from_job
-          self.class.assumible_attributes_from_job.each do |attribute|
+          self.class.assumable_attributes_from_job.each do |attribute|
             send("#{attribute}=", job.send(attribute))
           end
         end

data/app/models/solid_queue/execution.rb

@@ -2,6 +2,8 @@
 
 module SolidQueue
   class Execution < Record
+    class UndiscardableError < StandardError; end
+
     include JobAttributes
 
     self.abstract_class = true
@@ -10,8 +12,6 @@ module SolidQueue
 
     belongs_to :job
 
-    alias_method :discard, :destroy
-
     class << self
       def create_all_from_jobs(jobs)
         insert_all execution_data_from_jobs(jobs)
@@ -22,6 +22,48 @@ module SolidQueue
           attributes_from_job(job).merge(job_id: job.id)
         end
       end
+
+      def discard_all_in_batches(batch_size: 500)
+        pending = count
+        discarded = 0
+
+        loop do
+          transaction do
+            job_ids = limit(batch_size).order(:job_id).lock.pluck(:job_id)
+
+            discard_jobs job_ids
+            discarded = where(job_id: job_ids).delete_all
+            pending -= discarded
+          end
+
+          break if pending <= 0 || discarded == 0
+        end
+      end
+
+      def discard_all_from_jobs(jobs)
+        transaction do
+          job_ids = lock_all_from_jobs(jobs)
+
+          discard_jobs job_ids
+          where(job_id: job_ids).delete_all
+        end
+      end
+
+      private
+        def lock_all_from_jobs(jobs)
+          where(job_id: jobs.map(&:id)).order(:job_id).lock.pluck(:job_id)
+        end
+
+        def discard_jobs(job_ids)
+          Job.where(id: job_ids).delete_all
+        end
+    end
+
+    def discard
+      with_lock do
+        job.destroy
+        destroy
+      end
     end
   end
 end

data/app/models/solid_queue/failed_execution.rb

@@ -1,27 +1,37 @@
 # frozen_string_literal: true
 
-class SolidQueue::FailedExecution < SolidQueue::Execution
-  serialize :error, coder: JSON
+module SolidQueue
+  class FailedExecution < Execution
+    include Dispatching
 
-  before_create :expand_error_details_from_exception
+    serialize :error, coder: JSON
 
-  attr_accessor :exception
+    before_create :expand_error_details_from_exception
 
-  def retry
-    transaction do
-      job.prepare_for_execution
-      destroy!
+    attr_accessor :exception
+
+    def self.retry_all(jobs)
+      transaction do
+        dispatch_jobs lock_all_from_jobs(jobs)
+      end
+    end
+
+    def retry
+      with_lock do
+        job.prepare_for_execution
+        destroy!
+      end
     end
-  end
 
-  %i[ exception_class message backtrace ].each do |attribute|
-    define_method(attribute) { error.with_indifferent_access[attribute] }
-  end
+    %i[ exception_class message backtrace ].each do |attribute|
+      define_method(attribute) { error.with_indifferent_access[attribute] }
+    end
 
-  private
-    def expand_error_details_from_exception
-      if exception
-        self.error = { exception_class: exception.class.name, message: exception.message, backtrace: exception.backtrace }
+    private
+      def expand_error_details_from_exception
+        if exception
+          self.error = { exception_class: exception.class.name, message: exception.message, backtrace: exception.backtrace }
+        end
       end
     end
 end

data/app/models/solid_queue/job/clearable.rb

@@ -11,7 +11,10 @@ module SolidQueue
 
       class_methods do
         def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago)
-          clearable(finished_before: finished_before).in_batches(of: batch_size).delete_all
+          loop do
+            records_deleted = clearable(finished_before: finished_before).limit(batch_size).delete_all
+            break if records_deleted == 0
+          end
         end
       end
     end

data/app/models/solid_queue/job/concurrency_controls.rb

@@ -6,9 +6,17 @@ module SolidQueue
       extend ActiveSupport::Concern
 
       included do
-        has_one :blocked_execution, dependent: :destroy
+        has_one :blocked_execution
 
         delegate :concurrency_limit, :concurrency_duration, to: :job_class
+
+        before_destroy :unblock_next_blocked_job, if: -> { concurrency_limited? && ready? }
+      end
+
+      class_methods do
+        def release_all_concurrency_locks(jobs)
+          Semaphore.signal_all(jobs.select(&:concurrency_limited?))
+        end
       end
 
       def unblock_next_blocked_job
@@ -21,6 +29,10 @@ module SolidQueue
         concurrency_key.present?
       end
 
+      def blocked?
+        blocked_execution.present?
+      end
+
       private
         def acquire_concurrency_lock
           return true unless concurrency_limited?
@@ -45,6 +57,10 @@ module SolidQueue
         def job_class
           @job_class ||= class_name.safe_constantize
         end
+
+        def execution
+          super || blocked_execution
+        end
     end
   end
 end

data/app/models/solid_queue/job/executable.rb

@@ -8,9 +8,9 @@ module SolidQueue
       included do
         include Clearable, ConcurrencyControls, Schedulable
 
-        has_one :ready_execution, dependent: :destroy
-        has_one :claimed_execution, dependent: :destroy
-        has_one :failed_execution, dependent: :destroy
+        has_one :ready_execution
+        has_one :claimed_execution
+        has_one :failed_execution
 
         after_create :prepare_for_execution
 
@@ -73,6 +73,10 @@ module SolidQueue
         end
       end
 
+      def dispatch_bypassing_concurrency_limits
+        ready
+      end
+
       def finished!
         if preserve_finished_jobs?
           touch(:finished_at)
@@ -85,18 +89,22 @@ module SolidQueue
         finished_at.present?
       end
 
-      def due?
-        scheduled_at.nil? || scheduled_at <= Time.current
-      end
-
-      def discard
-        destroy unless claimed?
+      def status
+        if finished?
+          :finished
+        elsif execution.present?
+          execution.model_name.element.sub("_execution", "").to_sym
+        end
       end
 
       def retry
         failed_execution&.retry
       end
 
+      def discard
+        execution&.discard
+      end
+
       def failed_with(exception)
         FailedExecution.create_or_find_by!(job_id: id, exception: exception)
       end
@@ -106,6 +114,9 @@ module SolidQueue
           ReadyExecution.create_or_find_by!(job_id: id)
         end
 
+        def execution
+          %w[ ready claimed failed ].reduce(nil) { |acc, status| acc || public_send("#{status}_execution") }
+        end
 
         def preserve_finished_jobs?
           SolidQueue.preserve_finished_jobs

data/app/models/solid_queue/job/schedulable.rb

@@ -6,7 +6,7 @@ module SolidQueue
       extend ActiveSupport::Concern
 
       included do
-        has_one :scheduled_execution, dependent: :destroy
+        has_one :scheduled_execution
 
         scope :scheduled, -> { where.not(finished_at: nil) }
       end
@@ -31,10 +31,18 @@ module SolidQueue
         scheduled_at.nil? || scheduled_at <= Time.current
       end
 
+      def scheduled?
+        scheduled_execution.present?
+      end
+
       private
         def schedule
           ScheduledExecution.create_or_find_by!(job_id: id)
         end
+
+        def execution
+          super || scheduled_execution
+        end
     end
   end
 end

data/app/models/solid_queue/job.rb

@@ -15,9 +15,12 @@ module SolidQueue
           prepare_all_for_execution(jobs).tap do |enqueued_jobs|
             enqueued_jobs.each do |enqueued_job|
               active_jobs_by_job_id[enqueued_job.active_job_id].provider_job_id = enqueued_job.id
+              active_jobs_by_job_id[enqueued_job.active_job_id].successfully_enqueued = true
             end
           end
         end
+
+        active_jobs.count(&:successfully_enqueued?)
       end
 
       def enqueue(active_job, scheduled_at: Time.current)

data/app/models/solid_queue/queue.rb

@@ -33,7 +33,7 @@ module SolidQueue
     end
 
     def clear
-      Job.where(queue_name: name).each(&:discard)
+      ReadyExecution.queued_as(name).discard_all_in_batches
     end
 
     def size

data/app/models/solid_queue/ready_execution.rb

@@ -15,6 +15,10 @@ module SolidQueue
         end
       end
 
+      def aggregated_count_across(queue_list)
+        QueueSelector.new(queue_list, self).scoped_relations.map(&:count).sum
+      end
+
       private
         def select_and_lock(queue_relation, process_id, limit)
           return [] if limit <= 0
@@ -36,6 +40,12 @@ module SolidQueue
             where(job_id: claimed.pluck(:job_id)).delete_all
           end
         end
+
+
+        def discard_jobs(job_ids)
+          Job.release_all_concurrency_locks Job.where(id: job_ids)
+          super
+        end
     end
   end
 end

data/app/models/solid_queue/scheduled_execution.rb

@@ -2,6 +2,8 @@
 
 module SolidQueue
   class ScheduledExecution < Execution
+    include Dispatching
+
     scope :due, -> { where(scheduled_at: ..Time.current) }
     scope :ordered, -> { order(scheduled_at: :asc, priority: :asc) }
     scope :next_batch, ->(batch_size) { due.ordered.limit(batch_size) }
@@ -14,20 +16,10 @@ module SolidQueue
           job_ids = next_batch(batch_size).non_blocking_lock.pluck(:job_id)
           if job_ids.empty? then []
           else
-            dispatch_batch(job_ids)
+            dispatch_jobs(job_ids)
           end
         end
       end
-
-      private
-        def dispatch_batch(job_ids)
-          jobs = Job.where(id: job_ids)
-
-          Job.dispatch_all(jobs).map(&:id).tap do |dispatched_job_ids|
-            where(job_id: dispatched_job_ids).delete_all
-            SolidQueue.logger.info("[SolidQueue] Dispatched scheduled batch with #{dispatched_job_ids.size} jobs")
-          end
-        end
     end
   end
 end

data/app/models/solid_queue/semaphore.rb

@@ -1,65 +1,94 @@
 # frozen_string_literal: true
 
-class SolidQueue::Semaphore < SolidQueue::Record
-  scope :available, -> { where("value > 0") }
-  scope :expired, -> { where(expires_at: ...Time.current) }
+module SolidQueue
+  class Semaphore < Record
+    scope :available, -> { where("value > 0") }
+    scope :expired, -> { where(expires_at: ...Time.current) }
 
-  class << self
-    def wait(job)
-      Proxy.new(job, self).wait
-    end
-
-    def signal(job)
-      Proxy.new(job, self).signal
-    end
-  end
-
-  class Proxy
-    def initialize(job, proxied_class)
-      @job = job
-      @proxied_class = proxied_class
-    end
+    class << self
+      def wait(job)
+        Proxy.new(job).wait
+      end
 
-    def wait
-      if semaphore = proxied_class.find_by(key: key)
-        semaphore.value > 0 && attempt_decrement
-      else
-        attempt_creation
+      def signal(job)
+        Proxy.new(job).signal
       end
-    end
 
-    def signal
-      attempt_increment
+      def signal_all(jobs)
+        Proxy.signal_all(jobs)
+      end
     end
 
-    private
-      attr_reader :job, :proxied_class
-
-      def attempt_creation
-        proxied_class.create!(key: key, value: limit - 1, expires_at: expires_at)
-        true
-      rescue ActiveRecord::RecordNotUnique
-        attempt_decrement
+    class Proxy
+      def self.signal_all(jobs)
+        Semaphore.where(key: jobs.map(&:concurrency_key)).update_all("value = value + 1")
       end
 
-      def attempt_decrement
-        proxied_class.available.where(key: key).update_all([ "value = value - 1, expires_at = ?", expires_at ]) > 0
+      def initialize(job)
+        @job = job
+        @retries = 0
       end
 
-      def attempt_increment
-        proxied_class.where(key: key, value: ...limit).update_all([ "value = value + 1, expires_at = ?", expires_at ]) > 0
+      def wait
+        if semaphore = Semaphore.find_by(key: key)
+          semaphore.value > 0 && attempt_decrement
+        else
+          attempt_creation
+        end
       end
 
-      def key
-        job.concurrency_key
+      def signal
+        attempt_increment
       end
 
-      def expires_at
-        job.concurrency_duration.from_now
-      end
+      private
+        attr_accessor :job, :retries
 
-      def limit
-        job.concurrency_limit
-      end
+        def attempt_creation
+          Semaphore.create!(key: key, value: limit - 1, expires_at: expires_at)
+          true
+        rescue ActiveRecord::RecordNotUnique
+          attempt_decrement
+        end
+
+        def attempt_decrement
+          Semaphore.available.where(key: key).update_all([ "value = value - 1, expires_at = ?", expires_at ]) > 0
+        rescue ActiveRecord::Deadlocked
+          if retriable? then attempt_retry
+          else
+            raise
+          end
+        end
+
+        def attempt_increment
+          Semaphore.where(key: key, value: ...limit).update_all([ "value = value + 1, expires_at = ?", expires_at ]) > 0
+        end
+
+        def attempt_retry
+          self.retries += 1
+
+          if semaphore = Semaphore.find_by(key: key)
+            semaphore.value > 0 && attempt_decrement
+          end
+        end
+
+        MAX_RETRIES = 1
+
+        def retriable?
+          retries < MAX_RETRIES
+        end
+
+        def key
+          job.concurrency_key
+        end
+
+        def expires_at
+          job.concurrency_duration.from_now
+        end
+
+        def limit
+          job.concurrency_limit
+        end
+    end
   end
 end

data/db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb

@@ -0,0 +1,5 @@
+class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1]
+  def change
+    add_index :solid_queue_blocked_executions, [ :concurrency_key, :priority, :job_id ], name: "index_solid_queue_blocked_executions_for_release"
+  end
+end

data/lib/active_job/uniqueness.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Uniqueness
+    extend ActiveSupport::Concern
+
+    DEFAULT_UNIQUENESS_GROUP = ->(*) { self.class.name }
+
+    included do
+      class_attribute :uniqueness_key, instance_accessor: false
+      class_attribute :uniqueness_group, default: DEFAULT_UNIQUENESS_GROUP, instance_accessor: false
+
+      class_attribute :uniqueness_duration
+    end
+
+    class_methods do
+      def enqueued_uniquely_by(key:, group: DEFAULT_UNIQUENESS_GROUP, duration: SolidQueue.default_uniqueness_period)
+        self.uniqueness_key = key
+        self.uniqueness_group = group
+        self.uniqueness_duration = duration
+      end
+    end
+
+    def uniqueness_key
+      if self.class.uniqueness_key
+        param = compute_concurrency_parameter(self.class.concurrency_key)
+
+        case param
+        when ActiveRecord::Base
+          [ concurrency_group, param.class.name, param.id ]
+        else
+          [ concurrency_group, param ]
+        end.compact.join("/")
+      end
+    end
+
+    def enqueued_uniquely?
+      uniqueness_key.present?
+    end
+  end
+end

data/lib/generators/solid_queue/install/USAGE

@@ -4,6 +4,6 @@ Description:
 Example:
     bin/rails generate solid_queue:install
 
-    This will create:
+    This will perform the following:
         Installs solid_queue migrations
-        Replaces Active Job's adapter in envionment configuration
+        Replaces Active Job's adapter in environment configuration

data/lib/generators/solid_queue/install/templates/config.yml

@@ -1,4 +1,4 @@
-#default: &default
+# default: &default
 #   dispatchers:
 #     - polling_interval: 1
 #       batch_size: 500

data/lib/solid_queue/dispatcher/scheduled_executions_dispatcher.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Dispatcher::ScheduledExecutionsDispatcher < Dispatcher
+  end
+end

data/lib/solid_queue/processes/runnable.rb

@@ -4,6 +4,8 @@ module SolidQueue::Processes
   module Runnable
     include Supervised
 
+    attr_writer :mode
+
     def start
       @stopping = false
 
@@ -19,8 +21,6 @@ module SolidQueue::Processes
     end
 
   private
-    attr_writer :mode
-
     DEFAULT_MODE = :async
 
     def mode

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/solid_queue/worker.rb

@@ -44,7 +44,7 @@ module SolidQueue
       end
 
       def all_work_completed?
-        SolidQueue::ReadyExecution.queued_as(queues).empty?
+        SolidQueue::ReadyExecution.aggregated_count_across(queues).zero?
       end
 
       def metadata

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-26 00:00:00.000000000 Z
+date: 2024-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -79,6 +79,7 @@ files:
 - app/models/solid_queue/blocked_execution.rb
 - app/models/solid_queue/claimed_execution.rb
 - app/models/solid_queue/execution.rb
+- app/models/solid_queue/execution/dispatching.rb
 - app/models/solid_queue/execution/job_attributes.rb
 - app/models/solid_queue/failed_execution.rb
 - app/models/solid_queue/job.rb
@@ -97,8 +98,10 @@ files:
 - app/models/solid_queue/semaphore.rb
 - config/routes.rb
 - db/migrate/20231211200639_create_solid_queue_tables.rb
+- db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb
 - lib/active_job/concurrency_controls.rb
 - lib/active_job/queue_adapters/solid_queue_adapter.rb
+- lib/active_job/uniqueness.rb
 - lib/generators/solid_queue/install/USAGE
 - lib/generators/solid_queue/install/install_generator.rb
 - lib/generators/solid_queue/install/templates/config.yml
@@ -107,6 +110,7 @@ files:
 - lib/solid_queue/app_executor.rb
 - lib/solid_queue/configuration.rb
 - lib/solid_queue/dispatcher.rb
+- lib/solid_queue/dispatcher/scheduled_executions_dispatcher.rb
 - lib/solid_queue/engine.rb
 - lib/solid_queue/pool.rb
 - lib/solid_queue/processes/base.rb