solid_queue 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb47bf3ee9dffa1300c093abb260394d088f100b3c16a026d755f706d9f7a852
-  data.tar.gz: 10cc1b6f866d148d0c1cefce7587614aaca3e938f3f04370c15318e5b8d3509f
+  metadata.gz: 952d693063dae16e88a88acb2f8d77f396472cb29811e7ab2c47c06fc400cf10
+  data.tar.gz: 338ad6a0057939a54997cdef138fa5e62e974112625c43fd65240354a0ba760a
 SHA512:
-  metadata.gz: 71054017fcd26421f25140db9c929518ee011210fd6fea51e77352abf2a128289e965843dc1cb1fe97b62fa003d571e1de3631fcab329914b069ef9e0621806b
-  data.tar.gz: d08497fc98f9498aeaa3449182fc13cb90f71c859c67d55e0e5c14803bb1d5445e7717e1e3aca578fcb22a09c47126d465306e0220f54d6af4a8f8027fef0072
+  metadata.gz: b0bddd34b216770c9e0658bb620ae8801ab500074692417aeb0907c1f22f4f9e40a6233266aa69c1aa2b015dc294864d529659f0ee4adbdfdbef647d03fb4d35
+  data.tar.gz: f2323054f8fdc5ee686f738d2a8359b23fe88f00b1b376cc18e99588bacc9df3be2e3a8bb660ce31db854f0fcca91560ad4d703a737b1d05c3a1dea7817c10a2

data/app/jobs/solid_queue/recurring_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class SolidQueue::RecurringJob < ActiveJob::Base
+  def perform(command)
+    SolidQueue.instrument(:run_command, command: command) do
+      eval(command, TOPLEVEL_BINDING, __FILE__, __LINE__)
+    end
+  end
+end

data/app/models/solid_queue/recurring_execution.rb

@@ -7,16 +7,29 @@ module SolidQueue
     scope :clearable, -> { where.missing(:job) }
 
     class << self
+      def create_or_insert!(**attributes)
+        if connection.supports_insert_conflict_target?
+          # PostgreSQL fails and aborts the current transaction when it hits a duplicate key conflict
+          # during two concurrent INSERTs for the same value of an unique index. We need to explicitly
+          # indicate unique_by to ignore duplicate rows by this value when inserting
+          unless insert(attributes, unique_by: [ :task_key, :run_at ]).any?
+            raise AlreadyRecorded
+          end
+        else
+          create!(**attributes)
+        end
+      rescue ActiveRecord::RecordNotUnique
+        raise AlreadyRecorded
+      end
+
       def record(task_key, run_at, &block)
         transaction do
           block.call.tap do |active_job|
             if active_job
-              create!(job_id: active_job.provider_job_id, task_key: task_key, run_at: run_at)
+              create_or_insert!(job_id: active_job.provider_job_id, task_key: task_key, run_at: run_at)
             end
           end
         end
-      rescue ActiveRecord::RecordNotUnique => e
-        raise AlreadyRecorded
       end
 
       def clear_in_batches(batch_size: 500)

data/app/models/solid_queue/recurring_task/arguments.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "active_job/arguments"
+
+module SolidQueue
+  class RecurringTask::Arguments
+    class << self
+      def load(data)
+        data.nil? ? [] : ActiveJob::Arguments.deserialize(ActiveSupport::JSON.load(data))
+      end
+
+      def dump(data)
+        ActiveSupport::JSON.dump(ActiveJob::Arguments.serialize(Array(data)))
+      end
+    end
+  end
+end

data/{lib/solid_queue/dispatcher → app/models/solid_queue}/recurring_task.rb

@@ -1,24 +1,35 @@
+# frozen_string_literal: true
+
 require "fugit"
 
 module SolidQueue
-  class Dispatcher::RecurringTask
+  class RecurringTask < Record
+    serialize :arguments, coder: Arguments, default: []
+
+    validate :supported_schedule
+    validate :existing_job_class
+
+    scope :static, -> { where(static: true) }
+
     class << self
       def wrap(args)
         args.is_a?(self) ? args : from_configuration(args.first, **args.second)
       end
 
       def from_configuration(key, **options)
-        new(key, class_name: options[:class], schedule: options[:schedule], arguments: options[:args])
+        new(key: key, class_name: options[:class], schedule: options[:schedule], arguments: options[:args])
       end
-    end
 
-    attr_reader :key, :schedule, :class_name, :arguments
-
-    def initialize(key, class_name:, schedule:, arguments: nil)
-      @key = key
-      @class_name = class_name
-      @schedule = schedule
-      @arguments = Array(arguments)
+      def create_or_update_all(tasks)
+        if connection.supports_insert_conflict_target?
+          # PostgreSQL fails and aborts the current transaction when it hits a duplicate key conflict
+          # during two concurrent INSERTs for the same value of an unique index. We need to explicitly
+          # indicate unique_by to ignore duplicate rows by this value when inserting
+          upsert_all tasks.map(&:attributes_for_upsert), unique_by: :key
+        else
+          upsert_all tasks.map(&:attributes_for_upsert)
+        end
+      end
     end
 
     def delay_from_now
@@ -51,23 +62,27 @@ module SolidQueue
       end
     end
 
-    def valid?
-      parsed_schedule.instance_of?(Fugit::Cron)
-    end
-
     def to_s
       "#{class_name}.perform_later(#{arguments.map(&:inspect).join(",")}) [ #{parsed_schedule.original} ]"
     end
 
-    def to_h
-      {
-        schedule: schedule,
-        class_name: class_name,
-        arguments: arguments
-      }
+    def attributes_for_upsert
+      attributes.without("id", "created_at", "updated_at")
     end
 
     private
+      def supported_schedule
+        unless parsed_schedule.instance_of?(Fugit::Cron)
+          errors.add :schedule, :unsupported, message: "is not a supported recurring schedule"
+        end
+      end
+
+      def existing_job_class
+        unless job_class.present?
+          errors.add :class_name, :undefined, message: "doesn't correspond to an existing class"
+        end
+      end
+
       def using_solid_queue_adapter?
         job_class.queue_adapter_name.inquiry.solid_queue?
       end
@@ -88,12 +103,13 @@ module SolidQueue
         end
       end
 
+
       def parsed_schedule
         @parsed_schedule ||= Fugit.parse(schedule)
       end
 
       def job_class
-        @job_class ||= class_name.safe_constantize
+        @job_class ||= class_name&.safe_constantize
       end
   end
 end

data/app/models/solid_queue/semaphore.rb

@@ -17,6 +17,17 @@ module SolidQueue
       def signal_all(jobs)
         Proxy.signal_all(jobs)
       end
+
+      # Requires a unique index on key
+      def create_unique_by(attributes)
+        if connection.supports_insert_conflict_target?
+          insert({ **attributes }, unique_by: :key).any?
+        else
+          create!(**attributes)
+        end
+      rescue ActiveRecord::RecordNotUnique
+        false
+      end
     end
 
     class Proxy
@@ -44,15 +55,17 @@ module SolidQueue
         attr_accessor :job
 
         def attempt_creation
-          Semaphore.create!(key: key, value: limit - 1, expires_at: expires_at)
-          true
-        rescue ActiveRecord::RecordNotUnique
-          if limit == 1 then false
+          if Semaphore.create_unique_by(key: key, value: limit - 1, expires_at: expires_at)
+            true
           else
-            attempt_decrement
+            check_limit_or_decrement
           end
         end
 
+        def check_limit_or_decrement
+          limit == 1 ? false : attempt_decrement
+        end
+
         def attempt_decrement
           Semaphore.available.where(key: key).update_all([ "value = value - 1, expires_at = ?", expires_at ]) > 0
         end

data/db/migrate/20240719134516_create_recurring_tasks.rb

@@ -0,0 +1,20 @@
+class CreateRecurringTasks < ActiveRecord::Migration[7.1]
+  def change
+    create_table :solid_queue_recurring_tasks do |t|
+      t.string :key, null: false, index: { unique: true }
+      t.string :schedule, null: false
+      t.string :command, limit: 2048
+      t.string :class_name
+      t.text :arguments
+
+      t.string :queue_name
+      t.integer :priority, default: 0
+
+      t.boolean :static, default: true, index: true
+
+      t.text :description
+
+      t.timestamps
+    end
+  end
+end

data/lib/solid_queue/configuration.rb

@@ -75,7 +75,7 @@ module SolidQueue
 
       def parse_recurring_tasks(tasks)
         Array(tasks).map do |id, options|
-          Dispatcher::RecurringTask.from_configuration(id, **options)
+          RecurringTask.from_configuration(id, **options)
         end.select(&:valid?)
       end
 

data/lib/solid_queue/dispatcher/recurring_schedule.rb

@@ -7,7 +7,7 @@ module SolidQueue
     attr_reader :configured_tasks, :scheduled_tasks
 
     def initialize(tasks)
-      @configured_tasks = Array(tasks).map { |task| Dispatcher::RecurringTask.wrap(task) }
+      @configured_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?)
       @scheduled_tasks = Concurrent::Hash.new
     end
 
@@ -15,33 +15,42 @@ module SolidQueue
       configured_tasks.empty?
     end
 
-    def load_tasks
+    def schedule_tasks
+      wrap_in_app_executor do
+        persist_tasks
+        reload_tasks
+      end
+
       configured_tasks.each do |task|
-        load_task(task)
+        schedule_task(task)
       end
     end
 
-    def load_task(task)
+    def schedule_task(task)
       scheduled_tasks[task.key] = schedule(task)
     end
 
-    def unload_tasks
+    def unschedule_tasks
       scheduled_tasks.values.each(&:cancel)
       scheduled_tasks.clear
     end
 
-    def tasks
-      configured_tasks.each_with_object({}) { |task, hsh| hsh[task.key] = task.to_h }
-    end
-
-    def inspect
-      configured_tasks.map(&:to_s).join(" | ")
+    def task_keys
+      configured_tasks.map(&:key)
     end
 
     private
+      def persist_tasks
+        SolidQueue::RecurringTask.create_or_update_all configured_tasks
+      end
+
+      def reload_tasks
+        @configured_tasks = SolidQueue::RecurringTask.where(key: task_keys)
+      end
+
       def schedule(task)
         scheduled_task = Concurrent::ScheduledTask.new(task.delay_from_now, args: [ self, task, task.next_time ]) do |thread_schedule, thread_task, thread_task_run_at|
-          thread_schedule.load_task(thread_task)
+          thread_schedule.schedule_task(thread_task)
 
           wrap_in_app_executor do
             thread_task.enqueue(at: thread_task_run_at)

data/lib/solid_queue/dispatcher.rb

@@ -4,8 +4,8 @@ module SolidQueue
   class Dispatcher < Processes::Poller
     attr_accessor :batch_size, :concurrency_maintenance, :recurring_schedule
 
-    after_boot :start_concurrency_maintenance, :load_recurring_schedule
-    before_shutdown :stop_concurrency_maintenance, :unload_recurring_schedule
+    after_boot :start_concurrency_maintenance, :schedule_recurring_tasks
+    before_shutdown :stop_concurrency_maintenance, :unschedule_recurring_tasks
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
@@ -19,7 +19,7 @@ module SolidQueue
     end
 
     def metadata
-      super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.tasks.presence)
+      super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.task_keys.presence)
     end
 
     private
@@ -38,16 +38,16 @@ module SolidQueue
         concurrency_maintenance&.start
       end
 
-      def load_recurring_schedule
-        recurring_schedule.load_tasks
+      def schedule_recurring_tasks
+        recurring_schedule.schedule_tasks
       end
 
       def stop_concurrency_maintenance
         concurrency_maintenance&.stop
       end
 
-      def unload_recurring_schedule
-        recurring_schedule.unload_tasks
+      def unschedule_recurring_tasks
+        recurring_schedule.unschedule_tasks
       end
 
       def all_work_completed?

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-05 00:00:00.000000000 Z
+date: 2024-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -188,6 +188,7 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
+- app/jobs/solid_queue/recurring_job.rb
 - app/models/solid_queue/blocked_execution.rb
 - app/models/solid_queue/claimed_execution.rb
 - app/models/solid_queue/execution.rb
@@ -210,12 +211,15 @@ files:
 - app/models/solid_queue/ready_execution.rb
 - app/models/solid_queue/record.rb
 - app/models/solid_queue/recurring_execution.rb
+- app/models/solid_queue/recurring_task.rb
+- app/models/solid_queue/recurring_task/arguments.rb
 - app/models/solid_queue/scheduled_execution.rb
 - app/models/solid_queue/semaphore.rb
 - config/routes.rb
 - db/migrate/20231211200639_create_solid_queue_tables.rb
 - db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb
 - db/migrate/20240218110712_create_recurring_executions.rb
+- db/migrate/20240719134516_create_recurring_tasks.rb
 - lib/active_job/concurrency_controls.rb
 - lib/active_job/queue_adapters/solid_queue_adapter.rb
 - lib/generators/solid_queue/install/USAGE
@@ -228,7 +232,6 @@ files:
 - lib/solid_queue/dispatcher.rb
 - lib/solid_queue/dispatcher/concurrency_maintenance.rb
 - lib/solid_queue/dispatcher/recurring_schedule.rb
-- lib/solid_queue/dispatcher/recurring_task.rb
 - lib/solid_queue/engine.rb
 - lib/solid_queue/log_subscriber.rb
 - lib/solid_queue/pool.rb