que-scheduler 8.0.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/que/scheduler/audit.rb +5 -4
- data/lib/que/scheduler/db_support.rb +3 -8
- data/lib/que/scheduler/defined_job.rb +23 -80
- data/lib/que/scheduler/defined_job_parser.rb +96 -0
- data/lib/que/scheduler/enqueueing_calculator.rb +10 -7
- data/lib/que/scheduler/job_adapters.rb +122 -0
- data/lib/que/scheduler/schedule.rb +3 -31
- data/lib/que/scheduler/schedule_type.rb +17 -0
- data/lib/que/scheduler/scheduler_job.rb +8 -13
- data/lib/que/scheduler/scheduler_job_args.rb +17 -23
- data/lib/que/scheduler/to_enqueue.rb +20 -152
- data/lib/que/scheduler/types.rb +1 -1
- data/lib/que/scheduler/version.rb +1 -1
- metadata +4 -21
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 43c9761073fc43892c380b61ea7aeadb456c19b1f3d992eca44f93b7ec41c757
|
|
4
|
+
data.tar.gz: e7897e20ca62c7183f6db25289dc1781e825193238b1a0687eabe2ced6a3dcf5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3f5b2b777d5e52aff1d6422a84569b048db2b544e7b17457f1c413e5caa11d0505e90540a75264cd2fdecbdeb621a3f326f1591e5ff355169ecb1eba6b867d9d
|
|
7
|
+
data.tar.gz: 4b6ddb6b440fb761cecb48cdc815ecde9b1963535d2eb919b2f630ba899f54d6c21ace7c37a93258faa85c41f02164a0e52e1f8f5727a81e4148485b92811133
|
data/lib/que/scheduler/audit.rb
CHANGED
|
@@ -32,7 +32,7 @@ module Que
|
|
|
32
32
|
params(
|
|
33
33
|
scheduler_job_id: Integer,
|
|
34
34
|
executed_at: Types::TIME,
|
|
35
|
-
enqueued_jobs: T::Array[Que::Scheduler::
|
|
35
|
+
enqueued_jobs: T::Array[Que::Scheduler::EnqueuedJob]
|
|
36
36
|
).void
|
|
37
37
|
end
|
|
38
38
|
def append(scheduler_job_id, executed_at, enqueued_jobs)
|
|
@@ -40,10 +40,11 @@ module Que
|
|
|
40
40
|
enqueued_jobs.each do |j|
|
|
41
41
|
inserted = ::Que::Scheduler::DbSupport.execute(
|
|
42
42
|
INSERT_AUDIT_ENQUEUED,
|
|
43
|
-
[scheduler_job_id]
|
|
44
|
-
j.values_at(:job_class, :queue, :priority, :args, :job_id, :run_at)
|
|
43
|
+
[scheduler_job_id, j.job_class, j.queue, j.priority, j.args, j.job_id, j.run_at]
|
|
45
44
|
)
|
|
46
|
-
|
|
45
|
+
next unless inserted.empty?
|
|
46
|
+
|
|
47
|
+
raise "Cannot save audit row #{scheduler_job_id} #{executed_at} #{j.inspect}"
|
|
47
48
|
end
|
|
48
49
|
end
|
|
49
50
|
end
|
|
@@ -31,17 +31,12 @@ module Que
|
|
|
31
31
|
params(
|
|
32
32
|
clazz: T.class_of(Que::Job),
|
|
33
33
|
job_options: T::Hash[Symbol, T.anything],
|
|
34
|
-
job_args: T.anything
|
|
34
|
+
job_args: T::Array[T.anything]
|
|
35
35
|
).returns(T.nilable(T.any(Que::Job, FalseClass)))
|
|
36
36
|
end
|
|
37
37
|
def enqueue_a_job(clazz, job_options = {}, job_args = [])
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
clazz.enqueue(job_args, job_options: job_options)
|
|
41
|
-
else
|
|
42
|
-
# Sorbet cannot type a splat of a value of unknown size
|
|
43
|
-
T.unsafe(clazz).enqueue(*job_args, job_options: job_options)
|
|
44
|
-
end
|
|
38
|
+
# Sorbet cannot type a splat of an array of unknown size
|
|
39
|
+
T.unsafe(clazz).enqueue(*job_args, job_options: job_options)
|
|
45
40
|
end
|
|
46
41
|
|
|
47
42
|
sig { returns(String) }
|
|
@@ -1,39 +1,25 @@
|
|
|
1
1
|
# typed: true
|
|
2
2
|
|
|
3
|
-
require "hashie"
|
|
4
3
|
require "sorbet-runtime"
|
|
5
4
|
require "fugit"
|
|
5
|
+
require_relative "schedule_type"
|
|
6
|
+
require_relative "to_enqueue"
|
|
6
7
|
require_relative "types"
|
|
7
8
|
|
|
8
9
|
# This is the definition of one scheduleable job in the que-scheduler config yml file.
|
|
9
10
|
module Que
|
|
10
11
|
module Scheduler
|
|
11
|
-
class DefinedJob <
|
|
12
|
-
include Hashie::Extensions::Dash::PropertyTranslation
|
|
12
|
+
class DefinedJob < T::Struct
|
|
13
13
|
extend T::Sig
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
property :queue
|
|
24
|
-
property :priority
|
|
25
|
-
property :args_array
|
|
26
|
-
property :schedule_type, default: DEFINED_JOB_TYPE_DEFAULT
|
|
27
|
-
|
|
28
|
-
class << self
|
|
29
|
-
extend T::Sig
|
|
30
|
-
|
|
31
|
-
sig { params(options: T::Hash[Symbol, T.anything]).returns(DefinedJob) }
|
|
32
|
-
def create(options)
|
|
33
|
-
defined_job = new(options.compact)
|
|
34
|
-
defined_job.freeze.tap { |dj| dj.validate(options) }
|
|
35
|
-
end
|
|
36
|
-
end
|
|
15
|
+
const :name, String
|
|
16
|
+
const :job_class, Class
|
|
17
|
+
const :cron, Fugit::Cron
|
|
18
|
+
# nil means use the job class's own queue and priority
|
|
19
|
+
const :queue, T.nilable(String)
|
|
20
|
+
const :priority, T.nilable(Integer)
|
|
21
|
+
const :args_array, T::Array[T.anything], default: []
|
|
22
|
+
const :schedule_type, ScheduleType, default: ScheduleType::DEFAULT
|
|
37
23
|
|
|
38
24
|
# Given a "last time", return the next Time the event will occur, or nil if it
|
|
39
25
|
# is after "to".
|
|
@@ -58,67 +44,24 @@ module Que
|
|
|
58
44
|
generate_to_enqueue_list(missed_times)
|
|
59
45
|
end
|
|
60
46
|
|
|
61
|
-
sig { params(options: T::Hash[Symbol, T.anything]).void }
|
|
62
|
-
def validate(options)
|
|
63
|
-
validate_fields_presence(options)
|
|
64
|
-
validate_fields_types(options)
|
|
65
|
-
validate_job_class_related(options)
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
# rubocop:disable Style/GuardClause -- This reads better as a conditional
|
|
69
|
-
sig { params(options: T::Hash[Symbol, T.anything]).void }
|
|
70
|
-
private def validate_fields_types(options)
|
|
71
|
-
unless queue.nil? || queue.is_a?(String)
|
|
72
|
-
err_field(:queue, options, "queue must be a string")
|
|
73
|
-
end
|
|
74
|
-
unless priority.nil? || priority.is_a?(Integer)
|
|
75
|
-
err_field(:priority, options, "priority must be an integer")
|
|
76
|
-
end
|
|
77
|
-
unless DEFINED_JOB_TYPES.include?(schedule_type)
|
|
78
|
-
err_field(:schedule_type, options, "Not in #{DEFINED_JOB_TYPES}")
|
|
79
|
-
end
|
|
80
|
-
end
|
|
81
|
-
# rubocop:enable Style/GuardClause
|
|
82
|
-
|
|
83
|
-
sig { params(options: T::Hash[Symbol, T.anything]).void }
|
|
84
|
-
private def validate_fields_presence(options)
|
|
85
|
-
err_field(:name, options, "name must be present") if name.nil?
|
|
86
|
-
err_field(:job_class, options, "job_class must be present") if job_class.nil?
|
|
87
|
-
# An invalid cron is nil
|
|
88
|
-
err_field(:cron, options, "cron must be present") if cron.nil?
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
sig { params(options: T::Hash[Symbol, T.anything]).void }
|
|
92
|
-
private def validate_job_class_related(options)
|
|
93
|
-
# Only support known job engines
|
|
94
|
-
return if Que::Scheduler::ToEnqueue.valid_job_class?(job_class)
|
|
95
|
-
|
|
96
|
-
err_field(:job_class, options, "Job #{job_class} was not a supported job type")
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
sig { params(field: Symbol, options: T::Hash[Symbol, T.anything], reason: String).returns(T.noreturn) }
|
|
100
|
-
private def err_field(field, options, reason = "")
|
|
101
|
-
schedule = Que::Scheduler.configuration.schedule_location
|
|
102
|
-
value = options[field]
|
|
103
|
-
raise "Invalid #{field} '#{value}' for '#{name}' in que-scheduler schedule #{schedule}.\n" \
|
|
104
|
-
"#{reason}"
|
|
105
|
-
end
|
|
106
|
-
|
|
107
47
|
sig { params(missed_times: T::Array[ActiveSupport::TimeWithZone]).returns(T::Array[ToEnqueue]) }
|
|
108
48
|
private def generate_to_enqueue_list(missed_times)
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
options = to_h.slice(:args, :queue, :priority, :job_class).compact
|
|
49
|
+
last_missed_time = missed_times.last
|
|
50
|
+
return [] if last_missed_time.nil?
|
|
112
51
|
|
|
113
|
-
if schedule_type ==
|
|
114
|
-
missed_times.map
|
|
115
|
-
ToEnqueue.create(options.merge(run_at: time_missed,
|
|
116
|
-
args: [time_missed.iso8601] + args_array))
|
|
117
|
-
end
|
|
52
|
+
if schedule_type == ScheduleType::EVERY_EVENT
|
|
53
|
+
missed_times.map { to_enqueue(it, [it.iso8601] + args_array) }
|
|
118
54
|
else
|
|
119
|
-
[
|
|
55
|
+
[to_enqueue(last_missed_time, args_array)]
|
|
120
56
|
end
|
|
121
57
|
end
|
|
58
|
+
|
|
59
|
+
sig { params(run_at: ActiveSupport::TimeWithZone, args: T::Array[T.anything]).returns(ToEnqueue) }
|
|
60
|
+
private def to_enqueue(run_at, args)
|
|
61
|
+
ToEnqueue.new(
|
|
62
|
+
job_class: job_class, args: args, queue: queue, priority: priority, run_at: run_at
|
|
63
|
+
)
|
|
64
|
+
end
|
|
122
65
|
end
|
|
123
66
|
end
|
|
124
67
|
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
require "fugit"
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
require_relative "defined_job"
|
|
6
|
+
require_relative "job_adapters"
|
|
7
|
+
require_relative "schedule_type"
|
|
8
|
+
|
|
9
|
+
# Turns one job's entry in the schedule config into a validated DefinedJob.
|
|
10
|
+
module Que
|
|
11
|
+
module Scheduler
|
|
12
|
+
module DefinedJobParser
|
|
13
|
+
# One job's entry in the schedule, with string keys
|
|
14
|
+
CONFIG = T.type_alias { T::Hash[String, Object] }
|
|
15
|
+
|
|
16
|
+
class << self
|
|
17
|
+
extend T::Sig
|
|
18
|
+
|
|
19
|
+
sig { params(name: String, config: CONFIG).returns(DefinedJob) }
|
|
20
|
+
def parse(name, config)
|
|
21
|
+
DefinedJob.new(
|
|
22
|
+
name: name,
|
|
23
|
+
job_class: parse_job_class(name, config),
|
|
24
|
+
cron: parse_cron(name, config),
|
|
25
|
+
queue: parse_queue(name, config),
|
|
26
|
+
priority: parse_priority(name, config),
|
|
27
|
+
args_array: parse_args(config),
|
|
28
|
+
schedule_type: parse_schedule_type(name, config)
|
|
29
|
+
)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
sig { params(name: String, config: CONFIG).returns(Class) }
|
|
33
|
+
private def parse_job_class(name, config)
|
|
34
|
+
class_name = String(config.fetch("class", name))
|
|
35
|
+
job_class = Object.const_get(class_name)
|
|
36
|
+
return job_class if job_class.is_a?(Class) && JobAdapters.supports?(job_class)
|
|
37
|
+
|
|
38
|
+
reason = "Job #{class_name} was not a supported job type"
|
|
39
|
+
raise_invalid(name, :job_class, class_name, reason)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
sig { params(name: String, config: CONFIG).returns(Fugit::Cron) }
|
|
43
|
+
private def parse_cron(name, config)
|
|
44
|
+
value = config["cron"]
|
|
45
|
+
# An invalid cron parses to nil
|
|
46
|
+
Fugit::Cron.parse(value) || raise_invalid(name, :cron, value, "cron must be present")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
sig { params(name: String, config: CONFIG).returns(T.nilable(String)) }
|
|
50
|
+
private def parse_queue(name, config)
|
|
51
|
+
value = config["queue"]
|
|
52
|
+
return value if value.nil? || value.is_a?(String)
|
|
53
|
+
|
|
54
|
+
raise_invalid(name, :queue, value, "queue must be a string")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
sig { params(name: String, config: CONFIG).returns(T.nilable(Integer)) }
|
|
58
|
+
private def parse_priority(name, config)
|
|
59
|
+
value = config["priority"]
|
|
60
|
+
return value if value.nil? || value.is_a?(Integer)
|
|
61
|
+
|
|
62
|
+
raise_invalid(name, :priority, value, "priority must be an integer")
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Que stores args as a json array. A single value (including a hash or nil) becomes an
|
|
66
|
+
# array of that one value.
|
|
67
|
+
sig { params(config: CONFIG).returns(T::Array[T.anything]) }
|
|
68
|
+
private def parse_args(config)
|
|
69
|
+
return [] unless config.key?("args")
|
|
70
|
+
|
|
71
|
+
args = config["args"]
|
|
72
|
+
args.is_a?(Array) ? args : [args]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
sig { params(name: String, config: CONFIG).returns(ScheduleType) }
|
|
76
|
+
private def parse_schedule_type(name, config)
|
|
77
|
+
value = config["schedule_type"]
|
|
78
|
+
return ScheduleType::DEFAULT if value.nil?
|
|
79
|
+
|
|
80
|
+
reason = "Not in #{ScheduleType.values.map(&:serialize)}"
|
|
81
|
+
ScheduleType.try_deserialize(String(value)) ||
|
|
82
|
+
raise_invalid(name, :schedule_type, value, reason)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
sig do
|
|
86
|
+
params(name: String, field: Symbol, value: T.anything, reason: String).returns(T.noreturn)
|
|
87
|
+
end
|
|
88
|
+
private def raise_invalid(name, field, value, reason)
|
|
89
|
+
location = Que::Scheduler.configuration.schedule_location
|
|
90
|
+
raise "Invalid #{field} '#{value}' for '#{name}' " \
|
|
91
|
+
"in que-scheduler schedule #{location}.\n#{reason}"
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -2,13 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
require "fugit"
|
|
4
4
|
require "sorbet-runtime"
|
|
5
|
+
require_relative "defined_job"
|
|
6
|
+
require_relative "scheduler_job_args"
|
|
7
|
+
require_relative "to_enqueue"
|
|
5
8
|
|
|
6
9
|
module Que
|
|
7
10
|
module Scheduler
|
|
8
11
|
module EnqueueingCalculator
|
|
9
|
-
class Result <
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
class Result < T::Struct
|
|
13
|
+
const :missed_jobs, T::Array[ToEnqueue]
|
|
14
|
+
const :job_dictionary, T::Array[String]
|
|
12
15
|
end
|
|
13
16
|
|
|
14
17
|
class << self
|
|
@@ -21,22 +24,22 @@ module Que
|
|
|
21
24
|
).returns(Result)
|
|
22
25
|
end
|
|
23
26
|
def parse(scheduler_config, scheduler_job_args)
|
|
24
|
-
job_dictionary = []
|
|
27
|
+
job_dictionary = T.let([], T::Array[String])
|
|
25
28
|
|
|
26
29
|
# For each scheduled item:
|
|
27
30
|
# 1) If never seen before, schedule nothing, but add to known list
|
|
28
31
|
# 2) If seen before, calculate what (if anything) needs to be enqueued.
|
|
29
|
-
missed_jobs = scheduler_config.
|
|
32
|
+
missed_jobs = scheduler_config.flat_map do |desc|
|
|
30
33
|
job_name = desc.name
|
|
31
34
|
job_dictionary << job_name
|
|
32
35
|
|
|
33
36
|
# If we have never seen this job before, we don't want to scheduled any jobs for it.
|
|
34
37
|
# But we have added it to the dictionary, so it will be used to enqueue jobs next time.
|
|
35
|
-
next unless scheduler_job_args.job_dictionary.include?(job_name)
|
|
38
|
+
next [] unless scheduler_job_args.job_dictionary.include?(job_name)
|
|
36
39
|
|
|
37
40
|
# This has been seen before. We should check if we have missed any executions.
|
|
38
41
|
desc.calculate_missed_runs(scheduler_job_args.last_run_time, scheduler_job_args.as_time)
|
|
39
|
-
end
|
|
42
|
+
end
|
|
40
43
|
|
|
41
44
|
Result.new(missed_jobs: missed_jobs, job_dictionary: job_dictionary)
|
|
42
45
|
end
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "que"
|
|
5
|
+
require "sorbet-runtime"
|
|
6
|
+
require_relative "db_support"
|
|
7
|
+
require_relative "time_zone"
|
|
8
|
+
|
|
9
|
+
# Centralises the differences between enqueueing a Que::Job and an ActiveJob.
|
|
10
|
+
module Que
|
|
11
|
+
module Scheduler
|
|
12
|
+
module JobAdapter
|
|
13
|
+
extend T::Sig
|
|
14
|
+
extend T::Helpers
|
|
15
|
+
|
|
16
|
+
interface!
|
|
17
|
+
|
|
18
|
+
sig { abstract.params(to_enqueue: ToEnqueue).returns(T.nilable(EnqueuedJob)) }
|
|
19
|
+
def enqueue(to_enqueue); end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
class QueAdapter
|
|
23
|
+
extend T::Sig
|
|
24
|
+
include JobAdapter
|
|
25
|
+
|
|
26
|
+
sig { override.params(to_enqueue: ToEnqueue).returns(T.nilable(EnqueuedJob)) }
|
|
27
|
+
def enqueue(to_enqueue)
|
|
28
|
+
job_options = {
|
|
29
|
+
queue: to_enqueue.queue,
|
|
30
|
+
priority: to_enqueue.priority,
|
|
31
|
+
run_at: to_enqueue.run_at,
|
|
32
|
+
}.compact
|
|
33
|
+
job = DbSupport.enqueue_a_job(
|
|
34
|
+
T.cast(to_enqueue.job_class, T.class_of(Que::Job)), job_options, to_enqueue.args
|
|
35
|
+
)
|
|
36
|
+
read_back(job) if job
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Read the job back to get the values that will be used when it is worked
|
|
40
|
+
sig { params(job: Que::Job).returns(EnqueuedJob) }
|
|
41
|
+
private def read_back(job)
|
|
42
|
+
attrs = DbSupport.job_attributes(job)
|
|
43
|
+
EnqueuedJob.new(
|
|
44
|
+
job_class: T.cast(attrs.fetch(:job_class), String),
|
|
45
|
+
args: T.cast(attrs.fetch(:args), T::Array[T.anything]),
|
|
46
|
+
queue: T.cast(attrs.fetch(:queue), String),
|
|
47
|
+
priority: T.cast(attrs.fetch(:priority), Integer),
|
|
48
|
+
run_at: T.cast(attrs.fetch(:run_at), Time),
|
|
49
|
+
job_id: DbSupport.job_id(job)
|
|
50
|
+
)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
class ActiveJobAdapter
|
|
55
|
+
extend T::Sig
|
|
56
|
+
include JobAdapter
|
|
57
|
+
|
|
58
|
+
# Que's column default, which ActiveJob does not let us read back. t.ly/1jRK5
|
|
59
|
+
QUE_DEFAULT_PRIORITY = 100
|
|
60
|
+
|
|
61
|
+
sig { override.params(to_enqueue: ToEnqueue).returns(T.nilable(EnqueuedJob)) }
|
|
62
|
+
def enqueue(to_enqueue)
|
|
63
|
+
job_class = T.cast(to_enqueue.job_class, T.class_of(::ActiveJob::Base))
|
|
64
|
+
job_settings = {
|
|
65
|
+
priority: to_enqueue.priority,
|
|
66
|
+
wait_until: to_enqueue.run_at,
|
|
67
|
+
queue: to_enqueue.queue || Que::DEFAULT_QUEUE,
|
|
68
|
+
}.compact
|
|
69
|
+
# Sorbet cannot type a splat of an array of unknown size
|
|
70
|
+
job = T.unsafe(job_class.set(**job_settings)).perform_later(*to_enqueue.args)
|
|
71
|
+
read_back(job, to_enqueue) if job
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
sig { params(job: ::ActiveJob::Base, to_enqueue: ToEnqueue).returns(EnqueuedJob) }
|
|
75
|
+
private def read_back(job, to_enqueue)
|
|
76
|
+
data = JSON.parse(job.to_json, symbolize_names: true)
|
|
77
|
+
EnqueuedJob.new(
|
|
78
|
+
job_class: to_enqueue.job_class.to_s,
|
|
79
|
+
args: data.fetch(:arguments),
|
|
80
|
+
queue: data.fetch(:queue_name),
|
|
81
|
+
# The enqueued job does not expose its priority, so assume it is what we asked for
|
|
82
|
+
priority: to_enqueue.priority || QUE_DEFAULT_PRIORITY,
|
|
83
|
+
run_at: self.class.extract_scheduled_at(data.fetch(:scheduled_at)),
|
|
84
|
+
job_id: data.fetch(:provider_job_id)
|
|
85
|
+
)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# ActiveJob serialises scheduled_at as an ISO 8601 string; we want a Time
|
|
89
|
+
sig { params(scheduled_at: String).returns(ActiveSupport::TimeWithZone) }
|
|
90
|
+
def self.extract_scheduled_at(scheduled_at)
|
|
91
|
+
T.must(Que::Scheduler::TimeZone.time_zone.parse(scheduled_at))
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
module JobAdapters
|
|
96
|
+
QUE_ADAPTER = QueAdapter.new
|
|
97
|
+
ACTIVE_JOB_ADAPTER = ActiveJobAdapter.new
|
|
98
|
+
|
|
99
|
+
class << self
|
|
100
|
+
extend T::Sig
|
|
101
|
+
|
|
102
|
+
sig { returns(T::Boolean) }
|
|
103
|
+
def active_job_defined? = Object.const_defined?(:ActiveJob)
|
|
104
|
+
|
|
105
|
+
sig { params(job_class: Class).returns(T::Boolean) }
|
|
106
|
+
def supports?(job_class) = !!find(job_class)
|
|
107
|
+
|
|
108
|
+
sig { params(job_class: Class).returns(JobAdapter) }
|
|
109
|
+
def adapter_for(job_class) = T.must(find(job_class))
|
|
110
|
+
|
|
111
|
+
sig { params(job_class: Class).returns(T.nilable(JobAdapter)) }
|
|
112
|
+
private def find(job_class)
|
|
113
|
+
if job_class < ::Que::Job
|
|
114
|
+
QUE_ADAPTER
|
|
115
|
+
elsif active_job_defined? && job_class < ::ActiveJob::Base
|
|
116
|
+
ACTIVE_JOB_ADAPTER
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "sorbet-runtime"
|
|
4
4
|
require_relative "configuration"
|
|
5
5
|
require_relative "defined_job"
|
|
6
|
+
require_relative "defined_job_parser"
|
|
6
7
|
require_relative "types"
|
|
7
8
|
|
|
8
9
|
module Que
|
|
@@ -55,40 +56,11 @@ module Que
|
|
|
55
56
|
sig do
|
|
56
57
|
params(
|
|
57
58
|
name: String,
|
|
58
|
-
defined_job_hash_in: T::Hash[T.any(String, Symbol),
|
|
59
|
+
defined_job_hash_in: T::Hash[T.any(String, Symbol), Object]
|
|
59
60
|
).returns(DefinedJob)
|
|
60
61
|
end
|
|
61
62
|
def hash_item_to_defined_job(name, defined_job_hash_in)
|
|
62
|
-
|
|
63
|
-
# Que stores arguments as a json array. If the args we have to provide are already an
|
|
64
|
-
# array we can can simply pass them through. If it is a single non-nil value, then we make
|
|
65
|
-
# an array with one item which is that value (this includes if it is a hash). It could
|
|
66
|
-
# also be a single nil value.
|
|
67
|
-
args_array =
|
|
68
|
-
if defined_job_hash.key?("args")
|
|
69
|
-
args = defined_job_hash["args"]
|
|
70
|
-
if args.is_a?(Array)
|
|
71
|
-
# An array of args was requested
|
|
72
|
-
args
|
|
73
|
-
else
|
|
74
|
-
# A single value, a nil, or a hash was requested. que expects this to
|
|
75
|
-
# be enqueued as an array of 1 item
|
|
76
|
-
[args]
|
|
77
|
-
end
|
|
78
|
-
else
|
|
79
|
-
# No args were requested
|
|
80
|
-
[]
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
Que::Scheduler::DefinedJob.create(
|
|
84
|
-
name: name,
|
|
85
|
-
job_class: defined_job_hash["class"]&.to_s || name,
|
|
86
|
-
queue: defined_job_hash["queue"],
|
|
87
|
-
args_array: args_array,
|
|
88
|
-
priority: defined_job_hash["priority"],
|
|
89
|
-
cron: defined_job_hash["cron"],
|
|
90
|
-
schedule_type: defined_job_hash["schedule_type"]&.to_sym
|
|
91
|
-
)
|
|
63
|
+
DefinedJobParser.parse(name, defined_job_hash_in.stringify_keys)
|
|
92
64
|
end
|
|
93
65
|
end
|
|
94
66
|
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
require "sorbet-runtime"
|
|
4
|
+
|
|
5
|
+
module Que
|
|
6
|
+
module Scheduler
|
|
7
|
+
# How a job catches up on runs missed while the scheduler was not running.
|
|
8
|
+
class ScheduleType < T::Enum
|
|
9
|
+
enums do
|
|
10
|
+
# Enqueue only the most recent missed run
|
|
11
|
+
DEFAULT = new("default")
|
|
12
|
+
# Enqueue every missed run, passing each its scheduled time as the first arg
|
|
13
|
+
EVERY_EVENT = new("every_event")
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -30,7 +30,7 @@ module Que
|
|
|
30
30
|
self.maximum_retry_count = 1 << 128 # Heat death of universe
|
|
31
31
|
self.retry_interval = RETRY_PROC
|
|
32
32
|
|
|
33
|
-
sig { params(options: T.nilable(T::Hash[T.any(String, Symbol),
|
|
33
|
+
sig { params(options: T.nilable(T::Hash[T.any(String, Symbol), Object])).void }
|
|
34
34
|
def run(options = nil)
|
|
35
35
|
Que::Scheduler::Db.transaction do
|
|
36
36
|
Que::Scheduler::StateChecks.check
|
|
@@ -53,7 +53,7 @@ module Que
|
|
|
53
53
|
params(
|
|
54
54
|
calculator_result: EnqueueingCalculator::Result,
|
|
55
55
|
logs: T::Array[String]
|
|
56
|
-
).returns(T::Array[
|
|
56
|
+
).returns(T::Array[EnqueuedJob])
|
|
57
57
|
end
|
|
58
58
|
def enqueue_required_jobs(calculator_result, logs)
|
|
59
59
|
calculator_result.missed_jobs.filter_map do |to_enqueue|
|
|
@@ -66,7 +66,7 @@ module Que
|
|
|
66
66
|
sig do
|
|
67
67
|
params(
|
|
68
68
|
to_enqueue: ToEnqueue,
|
|
69
|
-
enqueued_job: T.nilable(
|
|
69
|
+
enqueued_job: T.nilable(EnqueuedJob),
|
|
70
70
|
logs: T::Array[String]
|
|
71
71
|
).void
|
|
72
72
|
end
|
|
@@ -86,7 +86,7 @@ module Que
|
|
|
86
86
|
scheduler_job_args: SchedulerJobArgs,
|
|
87
87
|
last_full_execution: Types::TIME,
|
|
88
88
|
job_dictionary: T::Array[String],
|
|
89
|
-
enqueued_jobs: T::Array[
|
|
89
|
+
enqueued_jobs: T::Array[EnqueuedJob]
|
|
90
90
|
).void
|
|
91
91
|
end
|
|
92
92
|
private def enqueue_self_again(scheduler_job_args, last_full_execution, job_dictionary,
|
|
@@ -103,17 +103,12 @@ module Que
|
|
|
103
103
|
queue: Que::Scheduler.configuration.que_scheduler_queue,
|
|
104
104
|
run_at: next_run_at,
|
|
105
105
|
},
|
|
106
|
-
{
|
|
107
|
-
last_run_time: last_full_execution.iso8601,
|
|
108
|
-
job_dictionary: job_dictionary,
|
|
109
|
-
}
|
|
106
|
+
[{ last_run_time: last_full_execution.iso8601, job_dictionary: job_dictionary }]
|
|
110
107
|
)
|
|
111
108
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
end
|
|
116
|
-
# rubocop:enable Style/GuardClause
|
|
109
|
+
return if enqueued_job
|
|
110
|
+
|
|
111
|
+
raise "SchedulerJob could not self-schedule. Has `.enqueue` been monkey patched?"
|
|
117
112
|
end
|
|
118
113
|
end
|
|
119
114
|
end
|
|
@@ -1,39 +1,33 @@
|
|
|
1
1
|
# typed: true
|
|
2
2
|
|
|
3
|
-
require "hashie"
|
|
4
3
|
require "sorbet-runtime"
|
|
5
4
|
require "active_support"
|
|
6
5
|
require "active_support/time_with_zone"
|
|
6
|
+
require_relative "types"
|
|
7
7
|
|
|
8
8
|
# These are the args that are used for this particular run of the scheduler.
|
|
9
9
|
module Que
|
|
10
10
|
module Scheduler
|
|
11
|
-
class SchedulerJobArgs <
|
|
12
|
-
property :last_run_time, required: true
|
|
13
|
-
property :job_dictionary, required: true
|
|
14
|
-
property :as_time, required: true
|
|
15
|
-
|
|
11
|
+
class SchedulerJobArgs < T::Struct
|
|
16
12
|
extend T::Sig
|
|
17
13
|
|
|
18
|
-
|
|
14
|
+
const :last_run_time, Types::TIME
|
|
15
|
+
const :job_dictionary, T::Array[String]
|
|
16
|
+
const :as_time, Types::TIME
|
|
17
|
+
|
|
18
|
+
sig { params(options: T.nilable(T::Hash[T.any(String, Symbol), Object])).returns(SchedulerJobArgs) }
|
|
19
19
|
def self.build(options)
|
|
20
20
|
now = Que::Scheduler::Db.now
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
last_run_time:
|
|
32
|
-
Que::Scheduler::TimeZone.time_zone.parse(options.fetch(:last_run_time)),
|
|
33
|
-
job_dictionary: options.fetch(:job_dictionary),
|
|
34
|
-
}
|
|
35
|
-
end
|
|
36
|
-
SchedulerJobArgs.new(parsed.merge(as_time: now))
|
|
21
|
+
# First ever run, there is nothing to do but reschedule self to run on the next minute.
|
|
22
|
+
return new(last_run_time: now, job_dictionary: [], as_time: now) if options.nil?
|
|
23
|
+
|
|
24
|
+
options = options.symbolize_keys
|
|
25
|
+
last_run_time = String(options.fetch(:last_run_time))
|
|
26
|
+
new(
|
|
27
|
+
last_run_time: T.must(Que::Scheduler::TimeZone.time_zone.parse(last_run_time)),
|
|
28
|
+
job_dictionary: T.cast(options.fetch(:job_dictionary), T::Array[String]),
|
|
29
|
+
as_time: now
|
|
30
|
+
)
|
|
37
31
|
end
|
|
38
32
|
end
|
|
39
33
|
end
|
|
@@ -1,166 +1,34 @@
|
|
|
1
1
|
# typed: true
|
|
2
2
|
|
|
3
|
-
require "que"
|
|
4
3
|
require "sorbet-runtime"
|
|
4
|
+
require_relative "job_adapters"
|
|
5
|
+
require_relative "types"
|
|
5
6
|
|
|
6
|
-
# This module uses polymorphic dispatch to centralise the differences between supporting Que::Job
|
|
7
|
-
# and other job systems.
|
|
8
7
|
module Que
|
|
9
8
|
module Scheduler
|
|
10
|
-
|
|
9
|
+
# A job the scheduler has decided to enqueue.
|
|
10
|
+
class ToEnqueue < T::Struct
|
|
11
11
|
extend T::Sig
|
|
12
|
-
extend T::Helpers
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
const :job_class, Class
|
|
14
|
+
const :args, T::Array[T.anything], default: []
|
|
15
|
+
# nil means use the job class's own queue and priority
|
|
16
|
+
const :queue, T.nilable(String)
|
|
17
|
+
const :priority, T.nilable(Integer)
|
|
18
|
+
const :run_at, Types::TIME
|
|
15
19
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
property :priority
|
|
19
|
-
property :run_at, required: true
|
|
20
|
-
property :job_class, required: true
|
|
21
|
-
|
|
22
|
-
sig { abstract.returns(T.nilable(EnqueuedJobType)) }
|
|
23
|
-
def enqueue; end
|
|
24
|
-
|
|
25
|
-
class << self
|
|
26
|
-
extend T::Sig
|
|
27
|
-
|
|
28
|
-
sig { params(options: T::Hash[Symbol, T.anything]).returns(ToEnqueue) }
|
|
29
|
-
def create(options)
|
|
30
|
-
T.must(type_from_job_class(T.cast(options.fetch(:job_class), Class))).new(
|
|
31
|
-
options.merge(run_at: options.fetch(:run_at) { Que::Scheduler::Db.now })
|
|
32
|
-
)
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
sig { params(job_class: Class).returns(T::Boolean) }
|
|
36
|
-
def valid_job_class?(job_class)
|
|
37
|
-
type_from_job_class(job_class).present?
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
sig { returns(T::Boolean) }
|
|
41
|
-
def active_job_defined?
|
|
42
|
-
Object.const_defined?(:ActiveJob)
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
sig { params(job_class: Class).returns(T.nilable(T.class_of(ToEnqueue))) }
|
|
46
|
-
private def type_from_job_class(job_class)
|
|
47
|
-
types.each do |type, implementation|
|
|
48
|
-
return implementation if job_class < type
|
|
49
|
-
end
|
|
50
|
-
nil
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
sig { returns(T::Hash[Class, T.class_of(ToEnqueue)]) }
|
|
54
|
-
private def types
|
|
55
|
-
@types ||=
|
|
56
|
-
begin
|
|
57
|
-
hash = {
|
|
58
|
-
::Que::Job => QueJobType,
|
|
59
|
-
}
|
|
60
|
-
hash[::ActiveJob::Base] = ActiveJobType if ToEnqueue.active_job_defined?
|
|
61
|
-
hash
|
|
62
|
-
end
|
|
63
|
-
end
|
|
64
|
-
end
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
# For jobs of type Que::Job
|
|
68
|
-
class QueJobType < ToEnqueue
|
|
69
|
-
sig { override.returns(T.nilable(EnqueuedJobType)) }
|
|
70
|
-
def enqueue
|
|
71
|
-
job_settings = to_h.slice(:queue, :priority, :run_at).compact
|
|
72
|
-
job = Que::Scheduler::DbSupport.enqueue_a_job(
|
|
73
|
-
T.cast(job_class, T.class_of(Que::Job)),
|
|
74
|
-
job_settings,
|
|
75
|
-
args
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
return nil if job.nil? || !job
|
|
79
|
-
|
|
80
|
-
# Now read the just inserted job back out of the DB to get the actual values that will
|
|
81
|
-
# be used when the job is worked.
|
|
82
|
-
values = Que::Scheduler::DbSupport.job_attributes(job).slice(
|
|
83
|
-
:args, :queue, :priority, :run_at, :job_class, :job_id
|
|
84
|
-
)
|
|
85
|
-
EnqueuedJobType.new(values)
|
|
86
|
-
end
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
# For jobs of type ActiveJob
|
|
90
|
-
class ActiveJobType < ToEnqueue
|
|
91
|
-
sig { override.returns(T.nilable(EnqueuedJobType)) }
|
|
92
|
-
def enqueue
|
|
93
|
-
job = enqueue_active_job
|
|
94
|
-
|
|
95
|
-
return nil if job.nil? || !job
|
|
96
|
-
|
|
97
|
-
enqueued_values = calculate_enqueued_values(job)
|
|
98
|
-
EnqueuedJobType.new(enqueued_values)
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
sig { params(job: ::ActiveJob::Base).returns(T::Hash[Symbol, T.anything]) }
|
|
102
|
-
def calculate_enqueued_values(job)
|
|
103
|
-
# Now read the just inserted job back out of the DB to get the actual values that will
|
|
104
|
-
# be used when the job is worked.
|
|
105
|
-
data = JSON.parse(job.to_json, symbolize_names: true)
|
|
106
|
-
|
|
107
|
-
scheduled_at = self.class.extract_scheduled_at(data[:scheduled_at])
|
|
108
|
-
|
|
109
|
-
# We can't get the priority out of the DB, as the returned `job` doesn't give us access
|
|
110
|
-
# to the underlying ActiveJob that was scheduled. We have no option but to assume
|
|
111
|
-
# it was what we told it to use. If no priority was specified, we must assume it was
|
|
112
|
-
# the Que default, which is 100 t.ly/1jRK5
|
|
113
|
-
assume_used_priority = priority.nil? ? 100 : priority
|
|
114
|
-
|
|
115
|
-
{
|
|
116
|
-
args: data.fetch(:arguments),
|
|
117
|
-
queue: data[:queue_name],
|
|
118
|
-
priority: assume_used_priority,
|
|
119
|
-
run_at: scheduled_at,
|
|
120
|
-
job_class: job_class.to_s,
|
|
121
|
-
job_id: data.fetch(:provider_job_id),
|
|
122
|
-
}
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
sig { returns(T.nilable(T.any(::ActiveJob::Base, FalseClass))) }
|
|
126
|
-
def enqueue_active_job
|
|
127
|
-
job_settings = {
|
|
128
|
-
priority: priority,
|
|
129
|
-
wait_until: run_at,
|
|
130
|
-
queue: queue || Que::DEFAULT_QUEUE,
|
|
131
|
-
}.compact
|
|
132
|
-
|
|
133
|
-
job_class_set = T.cast(job_class, T.class_of(::ActiveJob::Base)).set(**job_settings)
|
|
134
|
-
job_args = args
|
|
135
|
-
case job_args
|
|
136
|
-
when Hash
|
|
137
|
-
job_class_set.perform_later(**job_args)
|
|
138
|
-
else
|
|
139
|
-
# Sorbet cannot type a splat of a value of unknown size
|
|
140
|
-
T.unsafe(job_class_set).perform_later(*job_args)
|
|
141
|
-
end
|
|
142
|
-
end
|
|
143
|
-
|
|
144
|
-
class << self
|
|
145
|
-
extend T::Sig
|
|
146
|
-
|
|
147
|
-
# ActiveJob serialises scheduled_at as an ISO 8601 string; we want a Time
|
|
148
|
-
sig { params(scheduled_at: T.nilable(String)).returns(T.nilable(ActiveSupport::TimeWithZone)) }
|
|
149
|
-
def extract_scheduled_at(scheduled_at)
|
|
150
|
-
scheduled_at && Que::Scheduler::TimeZone.time_zone.parse(scheduled_at)
|
|
151
|
-
end
|
|
152
|
-
end
|
|
20
|
+
sig { returns(T.nilable(EnqueuedJob)) }
|
|
21
|
+
def enqueue = JobAdapters.adapter_for(job_class).enqueue(self)
|
|
153
22
|
end
|
|
154
23
|
|
|
155
|
-
# A
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
property :job_id, required: true
|
|
24
|
+
# A job as it was actually enqueued, read back after the enqueue.
|
|
25
|
+
class EnqueuedJob < T::Struct
|
|
26
|
+
const :job_class, String
|
|
27
|
+
const :args, T::Array[T.anything]
|
|
28
|
+
const :queue, String
|
|
29
|
+
const :priority, Integer
|
|
30
|
+
const :run_at, Types::TIME
|
|
31
|
+
const :job_id, Integer
|
|
164
32
|
end
|
|
165
33
|
end
|
|
166
34
|
end
|
data/lib/que/scheduler/types.rb
CHANGED
|
@@ -11,7 +11,7 @@ module Que
|
|
|
11
11
|
TIME = T.type_alias { T.any(Time, ActiveSupport::TimeWithZone) }
|
|
12
12
|
# A schedule as given in config: job name to its options
|
|
13
13
|
SCHEDULE_CONFIG = T.type_alias do
|
|
14
|
-
T::Hash[T.any(String, Symbol), T::Hash[T.any(String, Symbol),
|
|
14
|
+
T::Hash[T.any(String, Symbol), T::Hash[T.any(String, Symbol), Object]]
|
|
15
15
|
end
|
|
16
16
|
end
|
|
17
17
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: que-scheduler
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 9.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Harry Lascelles
|
|
@@ -43,26 +43,6 @@ dependencies:
|
|
|
43
43
|
- - ">="
|
|
44
44
|
- !ruby/object:Gem::Version
|
|
45
45
|
version: 1.11.1
|
|
46
|
-
- !ruby/object:Gem::Dependency
|
|
47
|
-
name: hashie
|
|
48
|
-
requirement: !ruby/object:Gem::Requirement
|
|
49
|
-
requirements:
|
|
50
|
-
- - ">="
|
|
51
|
-
- !ruby/object:Gem::Version
|
|
52
|
-
version: '3'
|
|
53
|
-
- - "<"
|
|
54
|
-
- !ruby/object:Gem::Version
|
|
55
|
-
version: '6'
|
|
56
|
-
type: :runtime
|
|
57
|
-
prerelease: false
|
|
58
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
59
|
-
requirements:
|
|
60
|
-
- - ">="
|
|
61
|
-
- !ruby/object:Gem::Version
|
|
62
|
-
version: '3'
|
|
63
|
-
- - "<"
|
|
64
|
-
- !ruby/object:Gem::Version
|
|
65
|
-
version: '6'
|
|
66
46
|
- !ruby/object:Gem::Dependency
|
|
67
47
|
name: que
|
|
68
48
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -398,7 +378,9 @@ files:
|
|
|
398
378
|
- lib/que/scheduler/db.rb
|
|
399
379
|
- lib/que/scheduler/db_support.rb
|
|
400
380
|
- lib/que/scheduler/defined_job.rb
|
|
381
|
+
- lib/que/scheduler/defined_job_parser.rb
|
|
401
382
|
- lib/que/scheduler/enqueueing_calculator.rb
|
|
383
|
+
- lib/que/scheduler/job_adapters.rb
|
|
402
384
|
- lib/que/scheduler/jobs/que_scheduler_audit_clear_down_job.rb
|
|
403
385
|
- lib/que/scheduler/migrations.rb
|
|
404
386
|
- lib/que/scheduler/migrations/1/down.sql
|
|
@@ -418,6 +400,7 @@ files:
|
|
|
418
400
|
- lib/que/scheduler/migrations/8/down.sql
|
|
419
401
|
- lib/que/scheduler/migrations/8/up.sql
|
|
420
402
|
- lib/que/scheduler/schedule.rb
|
|
403
|
+
- lib/que/scheduler/schedule_type.rb
|
|
421
404
|
- lib/que/scheduler/scheduler_job.rb
|
|
422
405
|
- lib/que/scheduler/scheduler_job_args.rb
|
|
423
406
|
- lib/que/scheduler/state_checks.rb
|