solid_queue 0.4.1 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (29) hide show
  1. checksums.yaml +4 -4
  2. data/app/jobs/solid_queue/recurring_job.rb +9 -0
  3. data/app/models/solid_queue/claimed_execution.rb +19 -7
  4. data/app/models/solid_queue/process/executor.rb +6 -0
  5. data/app/models/solid_queue/process/prunable.rb +14 -1
  6. data/app/models/solid_queue/process.rb +3 -5
  7. data/app/models/solid_queue/recurring_execution.rb +17 -4
  8. data/app/models/solid_queue/recurring_task/arguments.rb +17 -0
  9. data/app/models/solid_queue/recurring_task.rb +122 -0
  10. data/app/models/solid_queue/semaphore.rb +18 -5
  11. data/db/migrate/20240719134516_create_recurring_tasks.rb +20 -0
  12. data/db/migrate/20240811173327_add_name_to_processes.rb +5 -0
  13. data/db/migrate/20240813160053_make_name_not_null.rb +16 -0
  14. data/db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb +5 -0
  15. data/lib/solid_queue/configuration.rb +26 -20
  16. data/lib/solid_queue/dispatcher/recurring_schedule.rb +21 -12
  17. data/lib/solid_queue/dispatcher.rb +7 -7
  18. data/lib/solid_queue/log_subscriber.rb +13 -6
  19. data/lib/solid_queue/processes/base.rb +11 -0
  20. data/lib/solid_queue/processes/poller.rb +2 -0
  21. data/lib/solid_queue/processes/registrable.rb +1 -0
  22. data/lib/solid_queue/processes/runnable.rb +0 -4
  23. data/lib/solid_queue/supervisor/async_supervisor.rb +6 -3
  24. data/lib/solid_queue/supervisor/fork_supervisor.rb +39 -9
  25. data/lib/solid_queue/supervisor/maintenance.rb +11 -3
  26. data/lib/solid_queue/supervisor.rb +2 -1
  27. data/lib/solid_queue/version.rb +1 -1
  28. metadata +9 -3
  29. data/lib/solid_queue/dispatcher/recurring_task.rb +0 -99
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cb47bf3ee9dffa1300c093abb260394d088f100b3c16a026d755f706d9f7a852
4
- data.tar.gz: 10cc1b6f866d148d0c1cefce7587614aaca3e938f3f04370c15318e5b8d3509f
3
+ metadata.gz: e62ec09bbed1f2bcb96da96865c3516adbc254ad7cea234d3b6573ea001c758e
4
+ data.tar.gz: d78e7543194f3f6470bdb80c52ffca78c520a602d93423185c02ba82475bbda6
5
5
  SHA512:
6
- metadata.gz: 71054017fcd26421f25140db9c929518ee011210fd6fea51e77352abf2a128289e965843dc1cb1fe97b62fa003d571e1de3631fcab329914b069ef9e0621806b
7
- data.tar.gz: d08497fc98f9498aeaa3449182fc13cb90f71c859c67d55e0e5c14803bb1d5445e7717e1e3aca578fcb22a09c47126d465306e0220f54d6af4a8f8027fef0072
6
+ metadata.gz: e7cdceced9162911efdd0f2020042e45f7c73c74c506cf1d4958211b2cc34f02e96553d9910b3d3cca925f00e10960837be4872a33c3e44388e0b6c37cbcec31
7
+ data.tar.gz: d9a17282687ef9feaa4b357124411e9bd088bebb8c0ce6fd67ccf887e4d9cf20a431882e84b40f68ff5ca05a008258bf4d64b6ab86b675cb581be65154d1e39e
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ class SolidQueue::RecurringJob < ActiveJob::Base
4
+ def perform(command)
5
+ SolidQueue.instrument(:run_command, command: command) do
6
+ eval(command, TOPLEVEL_BINDING, __FILE__, __LINE__)
7
+ end
8
+ end
9
+ end
@@ -35,6 +35,18 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
35
35
  end
36
36
  end
37
37
 
38
+ def fail_all_with(error)
39
+ SolidQueue.instrument(:fail_many_claimed) do |payload|
40
+ includes(:job).tap do |executions|
41
+ payload[:size] = executions.size
42
+ payload[:process_ids] = executions.map(&:process_id).uniq
43
+ payload[:job_ids] = executions.map(&:job_id).uniq
44
+
45
+ executions.each { |execution| execution.failed_with(error) }
46
+ end
47
+ end
48
+ end
49
+
38
50
  def discard_all_in_batches(*)
39
51
  raise UndiscardableError, "Can't discard jobs in progress"
40
52
  end
@@ -69,6 +81,13 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
69
81
  raise UndiscardableError, "Can't discard a job in progress"
70
82
  end
71
83
 
84
+ def failed_with(error)
85
+ transaction do
86
+ job.failed_with(error)
87
+ destroy!
88
+ end
89
+ end
90
+
72
91
  private
73
92
  def execute
74
93
  ActiveJob::Base.execute(job.arguments)
@@ -83,11 +102,4 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
83
102
  destroy!
84
103
  end
85
104
  end
86
-
87
- def failed_with(error)
88
- transaction do
89
- job.failed_with(error)
90
- destroy!
91
- end
92
- end
93
105
  end
@@ -11,6 +11,12 @@ module SolidQueue
11
11
  after_destroy -> { claimed_executions.release_all }, if: :claims_executions?
12
12
  end
13
13
 
14
+ def fail_all_claimed_executions_with(error)
15
+ if claims_executions?
16
+ claimed_executions.fail_all_with(error)
17
+ end
18
+ end
19
+
14
20
  private
15
21
  def claims_executions?
16
22
  kind == "Worker"
@@ -1,6 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SolidQueue
4
+ class ProcessPrunedError < RuntimeError
5
+ def initialize(last_heartbeat_at)
6
+ super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
7
+ end
8
+ end
9
+
4
10
  class Process
5
11
  module Prunable
6
12
  extend ActiveSupport::Concern
@@ -15,11 +21,18 @@ module SolidQueue
15
21
  prunable.non_blocking_lock.find_in_batches(batch_size: 50) do |batch|
16
22
  payload[:size] += batch.size
17
23
 
18
- batch.each { |process| process.deregister(pruned: true) }
24
+ batch.each(&:prune)
19
25
  end
20
26
  end
21
27
  end
22
28
  end
29
+
30
+ def prune
31
+ error = ProcessPrunedError.new(last_heartbeat_at)
32
+ fail_all_claimed_executions_with(error)
33
+
34
+ deregister(pruned: true)
35
+ end
23
36
  end
24
37
  end
25
38
  end
@@ -13,10 +13,10 @@ class SolidQueue::Process < SolidQueue::Record
13
13
  create!(attributes.merge(last_heartbeat_at: Time.current)).tap do |process|
14
14
  payload[:process_id] = process.id
15
15
  end
16
+ rescue Exception => error
17
+ payload[:error] = error
18
+ raise
16
19
  end
17
- rescue Exception => error
18
- SolidQueue.instrument :register_process, **attributes.merge(error: error)
19
- raise
20
20
  end
21
21
 
22
22
  def heartbeat
@@ -25,8 +25,6 @@ class SolidQueue::Process < SolidQueue::Record
25
25
 
26
26
  def deregister(pruned: false)
27
27
  SolidQueue.instrument :deregister_process, process: self, pruned: pruned do |payload|
28
- payload[:claimed_size] = claimed_executions.size if claims_executions?
29
-
30
28
  destroy!
31
29
  rescue Exception => error
32
30
  payload[:error] = error
@@ -7,16 +7,29 @@ module SolidQueue
7
7
  scope :clearable, -> { where.missing(:job) }
8
8
 
9
9
  class << self
10
+ def create_or_insert!(**attributes)
11
+ if connection.supports_insert_conflict_target?
12
+ # PostgreSQL fails and aborts the current transaction when it hits a duplicate key conflict
13
+ # during two concurrent INSERTs for the same value of an unique index. We need to explicitly
14
+ # indicate unique_by to ignore duplicate rows by this value when inserting
15
+ unless insert(attributes, unique_by: [ :task_key, :run_at ]).any?
16
+ raise AlreadyRecorded
17
+ end
18
+ else
19
+ create!(**attributes)
20
+ end
21
+ rescue ActiveRecord::RecordNotUnique
22
+ raise AlreadyRecorded
23
+ end
24
+
10
25
  def record(task_key, run_at, &block)
11
26
  transaction do
12
27
  block.call.tap do |active_job|
13
- if active_job
14
- create!(job_id: active_job.provider_job_id, task_key: task_key, run_at: run_at)
28
+ if active_job && active_job.successfully_enqueued?
29
+ create_or_insert!(job_id: active_job.provider_job_id, task_key: task_key, run_at: run_at)
15
30
  end
16
31
  end
17
32
  end
18
- rescue ActiveRecord::RecordNotUnique => e
19
- raise AlreadyRecorded
20
33
  end
21
34
 
22
35
  def clear_in_batches(batch_size: 500)
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_job/arguments"
4
+
5
+ module SolidQueue
6
+ class RecurringTask::Arguments
7
+ class << self
8
+ def load(data)
9
+ data.nil? ? [] : ActiveJob::Arguments.deserialize(ActiveSupport::JSON.load(data))
10
+ end
11
+
12
+ def dump(data)
13
+ ActiveSupport::JSON.dump(ActiveJob::Arguments.serialize(Array(data)))
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fugit"
4
+
5
+ module SolidQueue
6
+ class RecurringTask < Record
7
+ serialize :arguments, coder: Arguments, default: []
8
+
9
+ validate :supported_schedule
10
+ validate :existing_job_class
11
+
12
+ scope :static, -> { where(static: true) }
13
+
14
+ class << self
15
+ def wrap(args)
16
+ args.is_a?(self) ? args : from_configuration(args.first, **args.second)
17
+ end
18
+
19
+ def from_configuration(key, **options)
20
+ new(key: key, class_name: options[:class], schedule: options[:schedule], arguments: options[:args])
21
+ end
22
+
23
+ def create_or_update_all(tasks)
24
+ if connection.supports_insert_conflict_target?
25
+ # PostgreSQL fails and aborts the current transaction when it hits a duplicate key conflict
26
+ # during two concurrent INSERTs for the same value of an unique index. We need to explicitly
27
+ # indicate unique_by to ignore duplicate rows by this value when inserting
28
+ upsert_all tasks.map(&:attributes_for_upsert), unique_by: :key
29
+ else
30
+ upsert_all tasks.map(&:attributes_for_upsert)
31
+ end
32
+ end
33
+ end
34
+
35
+ def delay_from_now
36
+ [ (next_time - Time.current).to_f, 0 ].max
37
+ end
38
+
39
+ def next_time
40
+ parsed_schedule.next_time.utc
41
+ end
42
+
43
+ def enqueue(at:)
44
+ SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload|
45
+ active_job = if using_solid_queue_adapter?
46
+ enqueue_and_record(run_at: at)
47
+ else
48
+ payload[:other_adapter] = true
49
+
50
+ perform_later do |job|
51
+ unless job.successfully_enqueued?
52
+ payload[:enqueue_error] = job.enqueue_error&.message
53
+ end
54
+ end
55
+ end
56
+
57
+ payload[:active_job_id] = active_job.job_id if active_job
58
+ rescue RecurringExecution::AlreadyRecorded
59
+ payload[:skipped] = true
60
+ rescue Job::EnqueueError => error
61
+ payload[:enqueue_error] = error.message
62
+ end
63
+ end
64
+
65
+ def to_s
66
+ "#{class_name}.perform_later(#{arguments.map(&:inspect).join(",")}) [ #{parsed_schedule.original} ]"
67
+ end
68
+
69
+ def attributes_for_upsert
70
+ attributes.without("id", "created_at", "updated_at")
71
+ end
72
+
73
+ private
74
+ def supported_schedule
75
+ unless parsed_schedule.instance_of?(Fugit::Cron)
76
+ errors.add :schedule, :unsupported, message: "is not a supported recurring schedule"
77
+ end
78
+ end
79
+
80
+ def existing_job_class
81
+ unless job_class.present?
82
+ errors.add :class_name, :undefined, message: "doesn't correspond to an existing class"
83
+ end
84
+ end
85
+
86
+ def using_solid_queue_adapter?
87
+ job_class.queue_adapter_name.inquiry.solid_queue?
88
+ end
89
+
90
+ def enqueue_and_record(run_at:)
91
+ RecurringExecution.record(key, run_at) do
92
+ job_class.new(*arguments_with_kwargs).tap do |active_job|
93
+ active_job.run_callbacks(:enqueue) do
94
+ Job.enqueue(active_job)
95
+ end
96
+ active_job.successfully_enqueued = true
97
+ end
98
+ end
99
+ end
100
+
101
+ def perform_later(&block)
102
+ job_class.perform_later(*arguments_with_kwargs, &block)
103
+ end
104
+
105
+ def arguments_with_kwargs
106
+ if arguments.last.is_a?(Hash)
107
+ arguments[0...-1] + [ Hash.ruby2_keywords_hash(arguments.last) ]
108
+ else
109
+ arguments
110
+ end
111
+ end
112
+
113
+
114
+ def parsed_schedule
115
+ @parsed_schedule ||= Fugit.parse(schedule)
116
+ end
117
+
118
+ def job_class
119
+ @job_class ||= class_name&.safe_constantize
120
+ end
121
+ end
122
+ end
@@ -17,6 +17,17 @@ module SolidQueue
17
17
  def signal_all(jobs)
18
18
  Proxy.signal_all(jobs)
19
19
  end
20
+
21
+ # Requires a unique index on key
22
+ def create_unique_by(attributes)
23
+ if connection.supports_insert_conflict_target?
24
+ insert({ **attributes }, unique_by: :key).any?
25
+ else
26
+ create!(**attributes)
27
+ end
28
+ rescue ActiveRecord::RecordNotUnique
29
+ false
30
+ end
20
31
  end
21
32
 
22
33
  class Proxy
@@ -44,15 +55,17 @@ module SolidQueue
44
55
  attr_accessor :job
45
56
 
46
57
  def attempt_creation
47
- Semaphore.create!(key: key, value: limit - 1, expires_at: expires_at)
48
- true
49
- rescue ActiveRecord::RecordNotUnique
50
- if limit == 1 then false
58
+ if Semaphore.create_unique_by(key: key, value: limit - 1, expires_at: expires_at)
59
+ true
51
60
  else
52
- attempt_decrement
61
+ check_limit_or_decrement
53
62
  end
54
63
  end
55
64
 
65
+ def check_limit_or_decrement
66
+ limit == 1 ? false : attempt_decrement
67
+ end
68
+
56
69
  def attempt_decrement
57
70
  Semaphore.available.where(key: key).update_all([ "value = value - 1, expires_at = ?", expires_at ]) > 0
58
71
  end
@@ -0,0 +1,20 @@
1
+ class CreateRecurringTasks < ActiveRecord::Migration[7.1]
2
+ def change
3
+ create_table :solid_queue_recurring_tasks do |t|
4
+ t.string :key, null: false, index: { unique: true }
5
+ t.string :schedule, null: false
6
+ t.string :command, limit: 2048
7
+ t.string :class_name
8
+ t.text :arguments
9
+
10
+ t.string :queue_name
11
+ t.integer :priority, default: 0
12
+
13
+ t.boolean :static, default: true, index: true
14
+
15
+ t.text :description
16
+
17
+ t.timestamps
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,5 @@
1
+ class AddNameToProcesses < ActiveRecord::Migration[7.1]
2
+ def change
3
+ add_column :solid_queue_processes, :name, :string
4
+ end
5
+ end
@@ -0,0 +1,16 @@
1
+ class MakeNameNotNull < ActiveRecord::Migration[7.1]
2
+ def up
3
+ SolidQueue::Process.where(name: nil).find_each do |process|
4
+ process.name ||= [ process.kind.downcase, SecureRandom.hex(10) ].join("-")
5
+ process.save!
6
+ end
7
+
8
+ change_column :solid_queue_processes, :name, :string, null: false
9
+ add_index :solid_queue_processes, [ :name, :supervisor_id ], unique: true
10
+ end
11
+
12
+ def down
13
+ remove_index :solid_queue_processes, [ :name, :supervisor_id ]
14
+ change_column :solid_queue_processes, :name, :string, null: false
15
+ end
16
+ end
@@ -0,0 +1,5 @@
1
+ class ChangeSolidQueueRecurringTasksStaticToNotNull < ActiveRecord::Migration[7.1]
2
+ def change
3
+ change_column_null :solid_queue_recurring_tasks, :static, false, true
4
+ end
5
+ end
@@ -2,6 +2,12 @@
2
2
 
3
3
  module SolidQueue
4
4
  class Configuration
5
+ class Process < Struct.new(:kind, :attributes)
6
+ def instantiate
7
+ "SolidQueue::#{kind.to_s.titleize}".safe_constantize.new(**attributes)
8
+ end
9
+ end
10
+
5
11
  WORKER_DEFAULTS = {
6
12
  queues: "*",
7
13
  threads: 3,
@@ -22,28 +28,10 @@ module SolidQueue
22
28
  @raw_config = config_from(load_from)
23
29
  end
24
30
 
25
- def processes
31
+ def configured_processes
26
32
  dispatchers + workers
27
33
  end
28
34
 
29
- def workers
30
- workers_options.flat_map do |worker_options|
31
- processes = if mode.fork?
32
- worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
33
- else
34
- WORKER_DEFAULTS[:processes]
35
- end
36
- processes.times.map { Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
37
- end
38
- end
39
-
40
- def dispatchers
41
- dispatchers_options.map do |dispatcher_options|
42
- recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
43
- Dispatcher.new **dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
44
- end
45
- end
46
-
47
35
  def max_number_of_threads
48
36
  # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
49
37
  workers_options.map { |options| options[:threads] }.max + 2
@@ -54,6 +42,24 @@ module SolidQueue
54
42
 
55
43
  DEFAULT_CONFIG_FILE_PATH = "config/solid_queue.yml"
56
44
 
45
+ def workers
46
+ workers_options.flat_map do |worker_options|
47
+ processes = if mode.fork?
48
+ worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
49
+ else
50
+ WORKER_DEFAULTS[:processes]
51
+ end
52
+ processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
53
+ end
54
+ end
55
+
56
+ def dispatchers
57
+ dispatchers_options.map do |dispatcher_options|
58
+ recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
59
+ Process.new :dispatcher, dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
60
+ end
61
+ end
62
+
57
63
  def config_from(file_or_hash, env: Rails.env)
58
64
  config = load_config_from(file_or_hash)
59
65
  config[env.to_sym] ? config[env.to_sym] : config
@@ -75,7 +81,7 @@ module SolidQueue
75
81
 
76
82
  def parse_recurring_tasks(tasks)
77
83
  Array(tasks).map do |id, options|
78
- Dispatcher::RecurringTask.from_configuration(id, **options)
84
+ RecurringTask.from_configuration(id, **options)
79
85
  end.select(&:valid?)
80
86
  end
81
87
 
@@ -7,7 +7,7 @@ module SolidQueue
7
7
  attr_reader :configured_tasks, :scheduled_tasks
8
8
 
9
9
  def initialize(tasks)
10
- @configured_tasks = Array(tasks).map { |task| Dispatcher::RecurringTask.wrap(task) }
10
+ @configured_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?)
11
11
  @scheduled_tasks = Concurrent::Hash.new
12
12
  end
13
13
 
@@ -15,33 +15,42 @@ module SolidQueue
15
15
  configured_tasks.empty?
16
16
  end
17
17
 
18
- def load_tasks
18
+ def schedule_tasks
19
+ wrap_in_app_executor do
20
+ persist_tasks
21
+ reload_tasks
22
+ end
23
+
19
24
  configured_tasks.each do |task|
20
- load_task(task)
25
+ schedule_task(task)
21
26
  end
22
27
  end
23
28
 
24
- def load_task(task)
29
+ def schedule_task(task)
25
30
  scheduled_tasks[task.key] = schedule(task)
26
31
  end
27
32
 
28
- def unload_tasks
33
+ def unschedule_tasks
29
34
  scheduled_tasks.values.each(&:cancel)
30
35
  scheduled_tasks.clear
31
36
  end
32
37
 
33
- def tasks
34
- configured_tasks.each_with_object({}) { |task, hsh| hsh[task.key] = task.to_h }
35
- end
36
-
37
- def inspect
38
- configured_tasks.map(&:to_s).join(" | ")
38
+ def task_keys
39
+ configured_tasks.map(&:key)
39
40
  end
40
41
 
41
42
  private
43
+ def persist_tasks
44
+ SolidQueue::RecurringTask.create_or_update_all configured_tasks
45
+ end
46
+
47
+ def reload_tasks
48
+ @configured_tasks = SolidQueue::RecurringTask.where(key: task_keys)
49
+ end
50
+
42
51
  def schedule(task)
43
52
  scheduled_task = Concurrent::ScheduledTask.new(task.delay_from_now, args: [ self, task, task.next_time ]) do |thread_schedule, thread_task, thread_task_run_at|
44
- thread_schedule.load_task(thread_task)
53
+ thread_schedule.schedule_task(thread_task)
45
54
 
46
55
  wrap_in_app_executor do
47
56
  thread_task.enqueue(at: thread_task_run_at)
@@ -4,8 +4,8 @@ module SolidQueue
4
4
  class Dispatcher < Processes::Poller
5
5
  attr_accessor :batch_size, :concurrency_maintenance, :recurring_schedule
6
6
 
7
- after_boot :start_concurrency_maintenance, :load_recurring_schedule
8
- before_shutdown :stop_concurrency_maintenance, :unload_recurring_schedule
7
+ after_boot :start_concurrency_maintenance, :schedule_recurring_tasks
8
+ before_shutdown :stop_concurrency_maintenance, :unschedule_recurring_tasks
9
9
 
10
10
  def initialize(**options)
11
11
  options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
@@ -19,7 +19,7 @@ module SolidQueue
19
19
  end
20
20
 
21
21
  def metadata
22
- super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.tasks.presence)
22
+ super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.task_keys.presence)
23
23
  end
24
24
 
25
25
  private
@@ -38,16 +38,16 @@ module SolidQueue
38
38
  concurrency_maintenance&.start
39
39
  end
40
40
 
41
- def load_recurring_schedule
42
- recurring_schedule.load_tasks
41
+ def schedule_recurring_tasks
42
+ recurring_schedule.schedule_tasks
43
43
  end
44
44
 
45
45
  def stop_concurrency_maintenance
46
46
  concurrency_maintenance&.stop
47
47
  end
48
48
 
49
- def unload_recurring_schedule
50
- recurring_schedule.unload_tasks
49
+ def unschedule_recurring_tasks
50
+ recurring_schedule.unschedule_tasks
51
51
  end
52
52
 
53
53
  def all_work_completed?
@@ -12,11 +12,15 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
12
12
  end
13
13
 
14
14
  def release_many_claimed(event)
15
- debug formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
15
+ info formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
16
+ end
17
+
18
+ def fail_many_claimed(event)
19
+ warn formatted_event(event, action: "Fail claimed jobs", **event.payload.slice(:job_ids, :process_ids))
16
20
  end
17
21
 
18
22
  def release_claimed(event)
19
- debug formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
23
+ info formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
20
24
  end
21
25
 
22
26
  def retry_all(event)
@@ -63,7 +67,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
63
67
  attributes = {
64
68
  pid: process.pid,
65
69
  hostname: process.hostname,
66
- process_id: process.process_id
70
+ process_id: process.process_id,
71
+ name: process.name
67
72
  }.merge(process.metadata)
68
73
 
69
74
  info formatted_event(event, action: "Started #{process.kind}", **attributes)
@@ -75,7 +80,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
75
80
  attributes = {
76
81
  pid: process.pid,
77
82
  hostname: process.hostname,
78
- process_id: process.process_id
83
+ process_id: process.process_id,
84
+ name: process.name
79
85
  }.merge(process.metadata)
80
86
 
81
87
  info formatted_event(event, action: "Shutdown #{process.kind}", **attributes)
@@ -83,7 +89,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
83
89
 
84
90
  def register_process(event)
85
91
  process_kind = event.payload[:kind]
86
- attributes = event.payload.slice(:pid, :hostname, :process_id)
92
+ attributes = event.payload.slice(:pid, :hostname, :process_id, :name)
87
93
 
88
94
  if error = event.payload[:error]
89
95
  warn formatted_event(event, action: "Error registering #{process_kind}", **attributes.merge(error: formatted_error(error)))
@@ -99,6 +105,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
99
105
  process_id: process.id,
100
106
  pid: process.pid,
101
107
  hostname: process.hostname,
108
+ name: process.name,
102
109
  last_heartbeat_at: process.last_heartbeat_at.iso8601,
103
110
  claimed_size: event.payload[:claimed_size],
104
111
  pruned: event.payload[:pruned]
@@ -147,7 +154,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
147
154
  termsig: status.termsig
148
155
 
149
156
  if replaced_fork = event.payload[:fork]
150
- info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname))
157
+ info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname, name: replaced_fork.name))
151
158
  else
152
159
  warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes)
153
160
  end
@@ -6,6 +6,12 @@ module SolidQueue
6
6
  include Callbacks # Defines callbacks needed by other concerns
7
7
  include AppExecutor, Registrable, Interruptible, Procline
8
8
 
9
+ attr_reader :name
10
+
11
+ def initialize(*)
12
+ @name = generate_name
13
+ end
14
+
9
15
  def kind
10
16
  self.class.name.demodulize
11
17
  end
@@ -21,6 +27,11 @@ module SolidQueue
21
27
  def metadata
22
28
  {}
23
29
  end
30
+
31
+ private
32
+ def generate_name
33
+ [ kind.downcase, SecureRandom.hex(10) ].join("-")
34
+ end
24
35
  end
25
36
  end
26
37
  end
@@ -8,6 +8,8 @@ module SolidQueue::Processes
8
8
 
9
9
  def initialize(polling_interval:, **options)
10
10
  @polling_interval = polling_interval
11
+
12
+ super(**options)
11
13
  end
12
14
 
13
15
  def metadata
@@ -21,6 +21,7 @@ module SolidQueue::Processes
21
21
  def register
22
22
  @process = SolidQueue::Process.register \
23
23
  kind: kind,
24
+ name: name,
24
25
  pid: pid,
25
26
  hostname: hostname,
26
27
  supervisor: try(:supervisor),
@@ -25,10 +25,6 @@ module SolidQueue::Processes
25
25
  @thread&.join
26
26
  end
27
27
 
28
- def name
29
- @name ||= [ kind.downcase, SecureRandom.hex(6) ].join("-")
30
- end
31
-
32
28
  def alive?
33
29
  !running_async? || @thread.alive?
34
30
  end
@@ -23,10 +23,13 @@ module SolidQueue
23
23
  attr_reader :threads
24
24
 
25
25
  def start_process(configured_process)
26
- configured_process.supervised_by process
27
- configured_process.start
26
+ process_instance = configured_process.instantiate.tap do |instance|
27
+ instance.supervised_by process
28
+ end
29
+
30
+ process_instance.start
28
31
 
29
- threads[configured_process.name] = configured_process
32
+ threads[process_instance.name] = process_instance
30
33
  end
31
34
 
32
35
  def stop_threads
@@ -1,12 +1,26 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SolidQueue
4
+ class ProcessExitError < RuntimeError
5
+ def initialize(status)
6
+ message = case
7
+ when status.exitstatus.present? then "Process pid=#{status.pid} exited with status #{status.exitstatus}"
8
+ when status.signaled? then "Process pid=#{status.pid} received unhandled signal #{status.termsig}"
9
+ else "Process pid=#{status.pid} exited unexpectedly"
10
+ end
11
+
12
+ super(message)
13
+ end
14
+ end
15
+
4
16
  class Supervisor::ForkSupervisor < Supervisor
5
17
  include Signals, Pidfiled
6
18
 
7
19
  def initialize(*)
8
20
  super
21
+
9
22
  @forks = {}
23
+ @configured_processes = {}
10
24
  end
11
25
 
12
26
  def kind
@@ -14,7 +28,7 @@ module SolidQueue
14
28
  end
15
29
 
16
30
  private
17
- attr_reader :forks
31
+ attr_reader :forks, :configured_processes
18
32
 
19
33
  def supervise
20
34
  loop do
@@ -33,14 +47,17 @@ module SolidQueue
33
47
  end
34
48
 
35
49
  def start_process(configured_process)
36
- configured_process.supervised_by process
37
- configured_process.mode = :fork
50
+ process_instance = configured_process.instantiate.tap do |instance|
51
+ instance.supervised_by process
52
+ instance.mode = :fork
53
+ end
38
54
 
39
55
  pid = fork do
40
- configured_process.start
56
+ process_instance.start
41
57
  end
42
58
 
43
- forks[pid] = configured_process
59
+ configured_processes[pid] = configured_process
60
+ forks[pid] = process_instance
44
61
  end
45
62
 
46
63
  def terminate_gracefully
@@ -86,7 +103,11 @@ module SolidQueue
86
103
  pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
87
104
  break unless pid
88
105
 
89
- forks.delete(pid)
106
+ if (terminated_fork = forks.delete(pid)) && !status.exited? || status.exitstatus > 0
107
+ handle_claimed_jobs_by(terminated_fork, status)
108
+ end
109
+
110
+ configured_processes.delete(pid)
90
111
  end
91
112
  rescue SystemCallError
92
113
  # All children already reaped
@@ -94,13 +115,22 @@ module SolidQueue
94
115
 
95
116
  def replace_fork(pid, status)
96
117
  SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
97
- if supervised_fork = forks.delete(pid)
98
- payload[:fork] = supervised_fork
99
- start_process(supervised_fork)
118
+ if terminated_fork = forks.delete(pid)
119
+ payload[:fork] = terminated_fork
120
+ handle_claimed_jobs_by(terminated_fork, status)
121
+
122
+ start_process(configured_processes.delete(pid))
100
123
  end
101
124
  end
102
125
  end
103
126
 
127
+ def handle_claimed_jobs_by(terminated_fork, status)
128
+ if registered_process = process.supervisees.find_by(name: terminated_fork.name)
129
+ error = ProcessExitError.new(status)
130
+ registered_process.fail_all_claimed_executions_with(error)
131
+ end
132
+ end
133
+
104
134
  def all_forks_terminated?
105
135
  forks.empty?
106
136
  end
@@ -1,9 +1,15 @@
1
1
  module SolidQueue
2
+ class ProcessMissingError < RuntimeError
3
+ def initialize
4
+ super("The process that was running this job no longer exists")
5
+ end
6
+ end
7
+
2
8
  module Supervisor::Maintenance
3
9
  extend ActiveSupport::Concern
4
10
 
5
11
  included do
6
- after_boot :release_orphaned_executions
12
+ after_boot :fail_orphaned_executions
7
13
  end
8
14
 
9
15
  private
@@ -27,8 +33,10 @@ module SolidQueue
27
33
  wrap_in_app_executor { SolidQueue::Process.prune }
28
34
  end
29
35
 
30
- def release_orphaned_executions
31
- wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
36
+ def fail_orphaned_executions
37
+ wrap_in_app_executor do
38
+ SolidQueue::ClaimedExecution.orphaned.fail_all_with(ProcessMissingError.new)
39
+ end
32
40
  end
33
41
  end
34
42
  end
@@ -16,6 +16,7 @@ module SolidQueue
16
16
 
17
17
  def initialize(configuration)
18
18
  @configuration = configuration
19
+ super
19
20
  end
20
21
 
21
22
  def start
@@ -44,7 +45,7 @@ module SolidQueue
44
45
  end
45
46
 
46
47
  def start_processes
47
- configuration.processes.each { |configured_process| start_process(configured_process) }
48
+ configuration.configured_processes.each { |configured_process| start_process(configured_process) }
48
49
  end
49
50
 
50
51
  def stopped?
@@ -1,3 +1,3 @@
1
1
  module SolidQueue
2
- VERSION = "0.4.1"
2
+ VERSION = "0.6.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: solid_queue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rosa Gutierrez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-08-05 00:00:00.000000000 Z
11
+ date: 2024-08-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -188,6 +188,7 @@ files:
188
188
  - MIT-LICENSE
189
189
  - README.md
190
190
  - Rakefile
191
+ - app/jobs/solid_queue/recurring_job.rb
191
192
  - app/models/solid_queue/blocked_execution.rb
192
193
  - app/models/solid_queue/claimed_execution.rb
193
194
  - app/models/solid_queue/execution.rb
@@ -210,12 +211,18 @@ files:
210
211
  - app/models/solid_queue/ready_execution.rb
211
212
  - app/models/solid_queue/record.rb
212
213
  - app/models/solid_queue/recurring_execution.rb
214
+ - app/models/solid_queue/recurring_task.rb
215
+ - app/models/solid_queue/recurring_task/arguments.rb
213
216
  - app/models/solid_queue/scheduled_execution.rb
214
217
  - app/models/solid_queue/semaphore.rb
215
218
  - config/routes.rb
216
219
  - db/migrate/20231211200639_create_solid_queue_tables.rb
217
220
  - db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb
218
221
  - db/migrate/20240218110712_create_recurring_executions.rb
222
+ - db/migrate/20240719134516_create_recurring_tasks.rb
223
+ - db/migrate/20240811173327_add_name_to_processes.rb
224
+ - db/migrate/20240813160053_make_name_not_null.rb
225
+ - db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb
219
226
  - lib/active_job/concurrency_controls.rb
220
227
  - lib/active_job/queue_adapters/solid_queue_adapter.rb
221
228
  - lib/generators/solid_queue/install/USAGE
@@ -228,7 +235,6 @@ files:
228
235
  - lib/solid_queue/dispatcher.rb
229
236
  - lib/solid_queue/dispatcher/concurrency_maintenance.rb
230
237
  - lib/solid_queue/dispatcher/recurring_schedule.rb
231
- - lib/solid_queue/dispatcher/recurring_task.rb
232
238
  - lib/solid_queue/engine.rb
233
239
  - lib/solid_queue/log_subscriber.rb
234
240
  - lib/solid_queue/pool.rb
@@ -1,99 +0,0 @@
1
- require "fugit"
2
-
3
- module SolidQueue
4
- class Dispatcher::RecurringTask
5
- class << self
6
- def wrap(args)
7
- args.is_a?(self) ? args : from_configuration(args.first, **args.second)
8
- end
9
-
10
- def from_configuration(key, **options)
11
- new(key, class_name: options[:class], schedule: options[:schedule], arguments: options[:args])
12
- end
13
- end
14
-
15
- attr_reader :key, :schedule, :class_name, :arguments
16
-
17
- def initialize(key, class_name:, schedule:, arguments: nil)
18
- @key = key
19
- @class_name = class_name
20
- @schedule = schedule
21
- @arguments = Array(arguments)
22
- end
23
-
24
- def delay_from_now
25
- [ (next_time - Time.current).to_f, 0 ].max
26
- end
27
-
28
- def next_time
29
- parsed_schedule.next_time.utc
30
- end
31
-
32
- def enqueue(at:)
33
- SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload|
34
- active_job = if using_solid_queue_adapter?
35
- perform_later_and_record(run_at: at)
36
- else
37
- payload[:other_adapter] = true
38
-
39
- perform_later do |job|
40
- unless job.successfully_enqueued?
41
- payload[:enqueue_error] = job.enqueue_error&.message
42
- end
43
- end
44
- end
45
-
46
- payload[:active_job_id] = active_job.job_id if active_job
47
- rescue RecurringExecution::AlreadyRecorded
48
- payload[:skipped] = true
49
- rescue Job::EnqueueError => error
50
- payload[:enqueue_error] = error.message
51
- end
52
- end
53
-
54
- def valid?
55
- parsed_schedule.instance_of?(Fugit::Cron)
56
- end
57
-
58
- def to_s
59
- "#{class_name}.perform_later(#{arguments.map(&:inspect).join(",")}) [ #{parsed_schedule.original} ]"
60
- end
61
-
62
- def to_h
63
- {
64
- schedule: schedule,
65
- class_name: class_name,
66
- arguments: arguments
67
- }
68
- end
69
-
70
- private
71
- def using_solid_queue_adapter?
72
- job_class.queue_adapter_name.inquiry.solid_queue?
73
- end
74
-
75
- def perform_later_and_record(run_at:)
76
- RecurringExecution.record(key, run_at) { perform_later }
77
- end
78
-
79
- def perform_later(&block)
80
- job_class.perform_later(*arguments_with_kwargs, &block)
81
- end
82
-
83
- def arguments_with_kwargs
84
- if arguments.last.is_a?(Hash)
85
- arguments[0...-1] + [ Hash.ruby2_keywords_hash(arguments.last) ]
86
- else
87
- arguments
88
- end
89
- end
90
-
91
- def parsed_schedule
92
- @parsed_schedule ||= Fugit.parse(schedule)
93
- end
94
-
95
- def job_class
96
- @job_class ||= class_name.safe_constantize
97
- end
98
- end
99
- end