solid_queue_mongoid 0.3.0

data/lib/solid_queue_mongoid/models/execution.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Execution < Record
+    include JobAttributes
+
+    class UndiscardableError < StandardError; end
+
+    scope :ordered, -> { order_by(priority: :asc, job_id: :asc) }
+
+    class << self
+      def type
+        model_name.element.sub("_execution", "").to_sym
+      end
+
+      def create_all_from_jobs(jobs)
+        jobs.each do |job|
+          attrs = attributes_from_job(job).merge(job_id: job.id)
+          next if where(job_id: job.id).exists?
+
+          create_or_find_by!(attrs)
+        end
+      end
+
+      def discard_all_in_batches(batch_size: 500)
+        pending = count
+        discarded = 0
+
+        loop do
+          job_ids = limit(batch_size).order_by(job_id: :asc).pluck(:job_id)
+          break if job_ids.empty?
+
+          discarded = Job.where(:id.in => job_ids).delete_all
+          where(:job_id.in => job_ids).delete_all
+          pending -= discarded
+
+          break if pending <= 0 || discarded.zero?
+        end
+      end
+
+      def discard_all_from_jobs(jobs)
+        job_ids = jobs.map(&:id)
+        discard_jobs(job_ids)
+        where(:job_id.in => job_ids).delete_all
+      end
+
+      private
+
+      def discard_jobs(job_ids)
+        Job.where(:id.in => job_ids).delete_all
+      end
+    end
+
+    def type
+      self.class.type
+    end
+
+    def discard
+      SolidQueue.instrument(:discard, job_id: job_id, status: type) do
+        job.destroy
+        destroy
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/failed_execution.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class FailedExecution < Execution
+    include Dispatching
+
+    field :error, type: Hash # stores exception_class, message, backtrace
+
+    attr_accessor :exception
+
+    before_save :expand_error_details_from_exception, if: :exception
+
+    index({ created_at: 1 })
+
+    class << self
+      def retry_all(jobs)
+        SolidQueue.instrument(:retry_all, jobs_size: jobs.size) do |payload|
+          job_ids = jobs.map(&:id)
+          payload[:size] = dispatch_jobs(lock_all_from_jobs_ids(job_ids))
+        end
+      end
+
+      private
+
+      def lock_all_from_jobs_ids(job_ids)
+        where(:job_id.in => job_ids).pluck(:job_id)
+      end
+    end
+
+    def retry
+      SolidQueue.instrument(:retry, job_id: job.id) do
+        Mongoid.transaction do
+          job.reset_execution_counters
+          job.prepare_for_execution
+          destroy!
+        end
+      end
+    end
+
+    # Error attribute accessors matching SolidQueue API
+    %i[exception_class message backtrace].each do |attribute|
+      define_method(attribute) { error&.with_indifferent_access&.[](attribute.to_s) }
+    end
+
+    private
+
+    # BSON documents are limited to 16 MB. Reserve a generous budget for the
+    # backtrace so a deep stack can never blow the limit.
+    BACKTRACE_SIZE_LIMIT = 50_000 # bytes of JSON
+
+    def expand_error_details_from_exception
+      return unless exception
+
+      self.error = {
+        "exception_class" => exception.class.name,
+        "message" => exception.message,
+        "backtrace" => truncate_backtrace(exception.backtrace)
+      }
+    end
+
+    def truncate_backtrace(lines)
+      return lines if lines.nil?
+
+      truncated = []
+      lines.each do |line|
+        truncated << line
+        break if truncated.to_json.bytesize > BACKTRACE_SIZE_LIMIT
+      end
+      # Remove the last line that pushed us over the limit
+      truncated.pop if truncated.to_json.bytesize > BACKTRACE_SIZE_LIMIT
+      truncated
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/job/clearable.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module Clearable
+      extend ActiveSupport::Concern
+
+      included do
+        scope :clearable, lambda { |finished_before: SolidQueue.clear_finished_jobs_after.ago, class_name: nil|
+          scope = where(:finished_at.ne => nil).where(:finished_at.lte => finished_before)
+          scope = scope.where(class_name: class_name) if class_name.present?
+          scope
+        }
+      end
+
+      class_methods do
+        def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago,
+                                      class_name: nil, sleep_between_batches: 0)
+          loop do
+            deleted = clearable(finished_before: finished_before, class_name: class_name).limit(batch_size).delete_all
+            sleep(sleep_between_batches) if sleep_between_batches.positive?
+            break if deleted.zero?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/job/concurrency_controls.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module ConcurrencyControls
+      extend ActiveSupport::Concern
+
+      included do
+        has_one :blocked_execution, class_name: "SolidQueue::BlockedExecution", dependent: :destroy
+
+        before_destroy :unblock_next_blocked_job, if: -> { concurrency_limited? && ready? }
+      end
+
+      class_methods do
+        def release_all_concurrency_locks(jobs)
+          Semaphore.signal_all(jobs.select(&:concurrency_limited?))
+        end
+      end
+
+      def unblock_next_blocked_job
+        return unless release_concurrency_lock
+
+        release_next_blocked_job
+      end
+
+      def concurrency_limited?
+        concurrency_key.present?
+      end
+
+      def blocked?
+        blocked_execution.present?
+      end
+
+      # Returns the concurrency limit: uses the stored field when set,
+      # otherwise delegates to job_class (AR-compatible behaviour).
+      def concurrency_limit
+        limit = read_attribute(:concurrency_limit)
+        return limit if limit.present?
+
+        job_class.respond_to?(:concurrency_limit) ? job_class.concurrency_limit : nil
+      end
+
+      # Returns the concurrency duration from job_class (not stored in DB).
+      def concurrency_duration
+        job_class.respond_to?(:concurrency_duration) ? job_class.concurrency_duration : 5.minutes
+      end
+
+      private
+
+      def concurrency_on_conflict
+        return "block".inquiry unless job_class.respond_to?(:concurrency_on_conflict)
+
+        job_class.concurrency_on_conflict.to_s.inquiry
+      end
+
+      def acquire_concurrency_lock
+        return true unless concurrency_limited?
+
+        Semaphore.wait(self)
+      end
+
+      def release_concurrency_lock
+        return false unless concurrency_limited?
+
+        Semaphore.signal(self)
+      end
+
+      def handle_concurrency_conflict
+        if concurrency_on_conflict.discard?
+          destroy
+        else
+          block
+        end
+      end
+
+      def block
+        BlockedExecution.create_or_find_by!(job_id: id)
+      end
+
+      def release_next_blocked_job
+        BlockedExecution.release_one(concurrency_key)
+      end
+
+      def job_class
+        @job_class ||= class_name.safe_constantize
+      end
+
+      def execution
+        super || blocked_execution
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/job/executable.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module Executable
+      extend ActiveSupport::Concern
+
+      included do
+        include ConcurrencyControls, Schedulable, Retryable
+
+        has_one :ready_execution,   class_name: "SolidQueue::ReadyExecution",   dependent: :destroy
+        has_one :claimed_execution, class_name: "SolidQueue::ClaimedExecution", dependent: :destroy
+
+        after_create :prepare_for_execution
+
+        scope :finished, -> { where(:finished_at.ne => nil) }
+        scope :failed,   -> { where(:id.in => SolidQueue::FailedExecution.all.pluck(:job_id)) }
+        scope :pending,  -> { where(finished_at: nil) }
+      end
+
+      class_methods do # rubocop:disable Metrics/BlockLength
+        # Dispatch a collection of jobs, partitioned by schedule and concurrency.
+        def prepare_all_for_execution(jobs)
+          due, not_yet_due = jobs.partition(&:due?)
+          dispatch_all(due) + schedule_all(not_yet_due)
+        end
+
+        def dispatch_all(jobs)
+          with_concurrency_limits, without_concurrency_limits = jobs.partition(&:concurrency_limited?)
+
+          dispatch_all_at_once(without_concurrency_limits)
+          dispatch_all_one_by_one(with_concurrency_limits)
+
+          successfully_dispatched(jobs)
+        end
+
+        private
+
+        def dispatch_all_at_once(jobs)
+          ReadyExecution.create_all_from_jobs(jobs)
+        end
+
+        def dispatch_all_one_by_one(jobs)
+          jobs.each(&:dispatch)
+        end
+
+        def successfully_dispatched(jobs)
+          jobs.map(&:id)
+          dispatched_and_ready(jobs) + dispatched_and_blocked(jobs)
+        end
+
+        def dispatched_and_ready(jobs)
+          job_ids = jobs.map(&:id)
+          where(:id.in => ReadyExecution.where(:job_id.in => job_ids).pluck(:job_id))
+        end
+
+        def dispatched_and_blocked(jobs)
+          job_ids = jobs.map(&:id)
+          where(:id.in => BlockedExecution.where(:job_id.in => job_ids).pluck(:job_id))
+        end
+      end
+
+      # status helpers matching SolidQueue runtime expectations
+      %w[ready claimed failed].each do |status|
+        define_method("#{status}?") { public_send("#{status}_execution").present? }
+      end
+
+      def prepare_for_execution
+        if due?
+          dispatch
+        else
+          schedule
+        end
+      end
+
+      def dispatch
+        if due?
+          if acquire_concurrency_lock
+            ready
+          else
+            handle_concurrency_conflict
+          end
+        else
+          schedule
+        end
+      end
+
+      # Called by ClaimedExecution#release — bypasses the semaphore check.
+      def dispatch_bypassing_concurrency_limits
+        ready
+      end
+
+      def finished!
+        if SolidQueue.preserve_finished_jobs?
+          update(finished_at: Time.current)
+          # Clean up the claimed execution if still present (e.g. called directly
+          # outside of ClaimedExecution#finished which does its own destroy!).
+          claimed_execution&.destroy
+        else
+          destroy!
+        end
+      end
+
+      alias finish finished!
+
+      def finished?
+        finished_at.present?
+      end
+
+      def status
+        if finished?
+          :finished
+        elsif (exec = execution)
+          exec.type
+        end
+      end
+
+      def discard
+        execution&.discard
+      end
+
+      def ready
+        existing = ReadyExecution.where(job_id: id).first
+        return existing if existing
+
+        re = ReadyExecution.new(job_id: id)
+        re.queue_name = queue_name
+        re.priority   = priority
+        re.save!
+        re
+      rescue Mongoid::Errors::Validations, Mongo::Error::OperationFailure
+        ReadyExecution.where(job_id: id).first
+      end
+
+      def execution
+        %w[ready claimed failed].reduce(nil) do |acc, status|
+          acc || public_send("#{status}_execution")
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/job/recurrable.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module Recurrable
+      extend ActiveSupport::Concern
+
+      included do
+        has_one :recurring_execution, class_name: "SolidQueue::RecurringExecution",
+                                      foreign_key: :job_id, dependent: :destroy
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/job/retryable.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module Retryable
+      extend ActiveSupport::Concern
+
+      included do
+        has_one :failed_execution, class_name: "SolidQueue::FailedExecution", dependent: :destroy
+      end
+
+      def retry
+        failed_execution&.retry
+      end
+
+      def can_retry?
+        max = read_attribute(:max_retries) || 0
+        count = read_attribute(:retry_count) || 0
+        count < max
+      end
+
+      def failed_with(exception)
+        FailedExecution.create!(job_id: id, exception: exception)
+      rescue Mongoid::Errors::Validations, Mongo::Error::OperationFailure => e
+        raise unless duplicate_key_error?(e)
+
+        existing = FailedExecution.find_by(job_id: id)
+        if existing
+          existing.exception = exception
+          existing.save!
+        else
+          retry
+        end
+      end
+
+      def reset_execution_counters
+        return unless arguments.is_a?(Hash)
+
+        arguments["executions"] = 0
+        arguments["exception_executions"] = {}
+        save!
+      end
+
+      private
+
+      def duplicate_key_error?(err)
+        err.message.to_s.include?("E11000") || err.message.to_s.include?("duplicate key")
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/job/schedulable.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module Schedulable
+      extend ActiveSupport::Concern
+
+      included do
+        field :scheduled_at, type: Time
+
+        index({ scheduled_at: 1 }, { sparse: true })
+
+        has_one :scheduled_execution, class_name: "SolidQueue::ScheduledExecution", dependent: :destroy
+
+        scope :scheduled, -> { where(finished_at: nil) }
+      end
+
+      class_methods do
+        def schedule_all(jobs)
+          schedule_all_at_once(jobs)
+          successfully_scheduled(jobs)
+        end
+
+        private
+
+        def schedule_all_at_once(jobs)
+          ScheduledExecution.create_all_from_jobs(jobs)
+        end
+
+        def successfully_scheduled(jobs)
+          job_ids = jobs.map(&:id)
+          where(:id.in => ScheduledExecution.where(:job_id.in => job_ids).pluck(:job_id))
+        end
+      end
+
+      # A job is due if it has no scheduled_at or it's in the past/present.
+      def due?
+        scheduled_at.nil? || scheduled_at <= Time.current
+      end
+
+      # True when a ScheduledExecution document exists.
+      def scheduled?
+        scheduled_execution.present?
+      end
+
+      def schedule
+        ScheduledExecution.create_or_find_by!(job_id: id)
+      end
+
+      def execution
+        super || scheduled_execution
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/job.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job < Record
+    class EnqueueError < StandardError; end
+
+    include Clearable
+    include Recurrable
+    include Executable # includes ConcurrencyControls, Schedulable, Retryable
+
+    field :queue_name, type: String
+    field :class_name, type: String
+    field :arguments, type: Hash, default: {}
+    field :priority, type: Integer, default: 0
+    field :active_job_id, type: String
+    field :concurrency_key, type: String
+    field :concurrency_limit, type: Integer # stored per-job; overrides job_class.concurrency_limit
+    field :finished_at, type: Time
+    field :max_retries, type: Integer, default: 0
+    field :retry_count, type: Integer, default: 0
+
+    index({ queue_name: 1 })
+    index({ class_name: 1 })
+    index({ priority: 1 })
+    index({ active_job_id: 1 }, { sparse: true })
+    index({ finished_at: 1 }, { sparse: true })
+    index({ concurrency_key: 1 }, { sparse: true })
+
+    validates :queue_name, :class_name, presence: true
+
+    DEFAULT_PRIORITY = 0
+    DEFAULT_QUEUE_NAME = "default"
+
+    class << self
+      # Primary enqueue entry point — called by the ActiveJob adapter.
+      def enqueue(active_job, scheduled_at: Time.current)
+        active_job.scheduled_at = scheduled_at
+
+        create_from_active_job(active_job).tap do |enqueued_job|
+          if enqueued_job.persisted?
+            active_job.provider_job_id = enqueued_job.id.to_s
+            active_job.successfully_enqueued = true
+          end
+        end
+      end
+
+      # Bulk enqueue — called by the ActiveJob adapter for perform_all_later.
+      def enqueue_all(active_jobs)
+        active_jobs.each { |job| job.scheduled_at = Time.current }
+        active_jobs_by_job_id = active_jobs.index_by(&:job_id)
+
+        jobs = create_all_from_active_jobs(active_jobs)
+
+        prepare_all_for_execution(jobs).tap do |enqueued_jobs|
+          enqueued_jobs.each do |enqueued_job|
+            aj = active_jobs_by_job_id[enqueued_job.active_job_id]
+            next unless aj
+
+            aj.provider_job_id = enqueued_job.id.to_s
+            aj.successfully_enqueued = true
+          end
+        end
+
+        active_jobs.count(&:successfully_enqueued?)
+      end
+
+      private
+
+      def create_from_active_job(active_job)
+        create!(**attributes_from_active_job(active_job))
+      rescue StandardError => e
+        enqueue_error = EnqueueError.new("#{e.class.name}: #{e.message}").tap do |err|
+          err.set_backtrace(e.backtrace)
+        end
+        raise enqueue_error
+      end
+
+      def create_all_from_active_jobs(active_jobs)
+        active_jobs.filter_map do |active_job|
+          create_from_active_job(active_job)
+        rescue EnqueueError
+          nil
+        end
+      end
+
+      def attributes_from_active_job(active_job)
+        {
+          queue_name: active_job.queue_name || DEFAULT_QUEUE_NAME,
+          active_job_id: active_job.job_id,
+          priority: active_job.priority || DEFAULT_PRIORITY,
+          scheduled_at: active_job.scheduled_at,
+          class_name: active_job.class.name,
+          arguments: active_job.serialize,
+          concurrency_key: active_job.try(:concurrency_key)
+        }.compact
+      end
+    end
+
+    def deserialize_for_active_job
+      ActiveJob::Base.deserialize(arguments)
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/pause.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Pause < Record
+    field :queue_name, type: String
+
+    index({ queue_name: 1 }, { unique: true })
+
+    validates :queue_name, presence: true, uniqueness: true
+
+    class << self
+      def pause_queue(queue_name)
+        create_or_find_by!(queue_name: queue_name)
+      end
+
+      def resume_queue(queue_name)
+        where(queue_name: queue_name).delete_all
+      end
+
+      def paused?(queue_name)
+        where(queue_name: queue_name).exists?
+      end
+    end
+  end
+end

data/lib/solid_queue_mongoid/models/process/executor.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Process
+    module Executor
+      extend ActiveSupport::Concern
+
+      included do
+        has_many :claimed_executions, class_name: "SolidQueue::ClaimedExecution",
+                                      foreign_key: :process_id
+
+        after_destroy :release_all_claimed_executions
+      end
+
+      def fail_all_claimed_executions_with(error)
+        claimed_executions.fail_all_with(error) if claims_executions?
+      end
+
+      def release_all_claimed_executions
+        claimed_executions.release_all if claims_executions?
+      end
+
+      private
+
+      def claims_executions?
+        kind.nil? || kind == "Worker"
+      end
+    end
+  end
+end