que-scheduler 3.2.8 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/lib/que/scheduler/audit.rb +3 -2
- data/lib/que/scheduler/defined_job.rb +72 -27
- data/lib/que/scheduler/schedule.rb +9 -11
- data/lib/que/scheduler/scheduler_job.rb +14 -23
- data/lib/que/scheduler/to_enqueue.rb +150 -0
- data/lib/que/scheduler/version.rb +1 -1
- metadata +12 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5707036df23117e995d696388d17bfc0488eef9c7ba30211b1a3700e6f0fb94f
|
4
|
+
data.tar.gz: 3133e78529f3658c9509e0a1240189424caefc8219b7da035530e68fdbfb38c6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4609167d5adfc2e7b9c583d3cbece89dd34e8624512b0c8c257804c05934964a343336b192944ce2790fde4f4b6029173df2f76d28fe7daee72e0a1ae473bad2
|
7
|
+
data.tar.gz: d55658a7923c50010a8a38713dc25c0f08d182004cb5156a336921f55b35e91631edb042f5e76f194d58ed04245ea8f08c78dd64386c2479ee4bdb8ba9a4512a
|
data/README.md
CHANGED
data/lib/que/scheduler/audit.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative 'to_enqueue'
|
4
|
+
|
3
5
|
module Que
|
4
6
|
module Scheduler
|
5
7
|
module Audit
|
@@ -24,11 +26,10 @@ module Que
|
|
24
26
|
def append(scheduler_job_id, executed_at, enqueued_jobs)
|
25
27
|
::Que::Scheduler::VersionSupport.execute(INSERT_AUDIT, [scheduler_job_id, executed_at])
|
26
28
|
enqueued_jobs.each do |j|
|
27
|
-
attrs = Que::Scheduler::VersionSupport.job_attributes(j)
|
28
29
|
inserted = ::Que::Scheduler::VersionSupport.execute(
|
29
30
|
INSERT_AUDIT_ENQUEUED,
|
30
31
|
[scheduler_job_id] +
|
31
|
-
|
32
|
+
j.values_at(:job_class, :queue, :priority, :args, :job_id, :run_at)
|
32
33
|
)
|
33
34
|
raise "Cannot save audit row #{scheduler_job_id} #{executed_at} #{j}" if inserted.empty?
|
34
35
|
end
|
@@ -10,23 +10,23 @@ module Que
|
|
10
10
|
|
11
11
|
DEFINED_JOB_TYPES = [
|
12
12
|
DEFINED_JOB_TYPE_DEFAULT = :default,
|
13
|
-
DEFINED_JOB_TYPE_EVERY_EVENT = :every_event
|
13
|
+
DEFINED_JOB_TYPE_EVERY_EVENT = :every_event,
|
14
14
|
].freeze
|
15
15
|
|
16
|
-
property :name
|
17
|
-
property :job_class,
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
property :cron, required: true, transform_with: lambda { |v|
|
22
|
-
Fugit::Cron.parse(v) || err_field(:cron, v)
|
23
|
-
}
|
24
|
-
property :queue, transform_with: ->(v) { v.is_a?(String) ? v : err_field(:queue, v) }
|
25
|
-
property :priority, transform_with: ->(v) { v.is_a?(Integer) ? v : err_field(:priority, v) }
|
16
|
+
property :name
|
17
|
+
property :job_class, transform_with: ->(v) { Object.const_get(v) }
|
18
|
+
property :cron, transform_with: ->(v) { Fugit::Cron.parse(v) }
|
19
|
+
property :queue
|
20
|
+
property :priority
|
26
21
|
property :args
|
27
|
-
property :schedule_type, default: DEFINED_JOB_TYPE_DEFAULT
|
28
|
-
|
29
|
-
|
22
|
+
property :schedule_type, default: DEFINED_JOB_TYPE_DEFAULT
|
23
|
+
|
24
|
+
class << self
|
25
|
+
def create(options)
|
26
|
+
defined_job = new(options.compact)
|
27
|
+
defined_job.freeze.tap { |dj| dj.validate(options) }
|
28
|
+
end
|
29
|
+
end
|
30
30
|
|
31
31
|
# Given a "last time", return the next Time the event will occur, or nil if it
|
32
32
|
# is after "to".
|
@@ -49,22 +49,67 @@ module Que
|
|
49
49
|
generate_to_enqueue_list(missed_times)
|
50
50
|
end
|
51
51
|
|
52
|
-
|
53
|
-
|
52
|
+
def validate(options)
|
53
|
+
validate_fields_presence(options)
|
54
|
+
validate_fields_types(options)
|
55
|
+
validate_job_class_related(options)
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
54
59
|
|
55
|
-
|
56
|
-
|
57
|
-
|
60
|
+
# rubocop:disable Style/GuardClause This reads better as a conditional
|
61
|
+
def validate_fields_types(options)
|
62
|
+
unless queue.nil? || queue.is_a?(String)
|
63
|
+
err_field(:queue, options, 'queue must be a string')
|
64
|
+
end
|
65
|
+
unless priority.nil? || priority.is_a?(Integer)
|
66
|
+
err_field(:priority, options, 'priority must be an integer')
|
67
|
+
end
|
68
|
+
unless DEFINED_JOB_TYPES.include?(schedule_type)
|
69
|
+
err_field(:schedule_type, options, "Not in #{DEFINED_JOB_TYPES}")
|
58
70
|
end
|
59
71
|
end
|
72
|
+
# rubocop:enable Style/GuardClause
|
60
73
|
|
61
|
-
|
74
|
+
def validate_fields_presence(options)
|
75
|
+
err_field(:name, options, 'name must be present') if name.nil?
|
76
|
+
err_field(:job_class, options, 'job_class must be present') if job_class.nil?
|
77
|
+
# An invalid cron is nil
|
78
|
+
err_field(:cron, options, 'cron must be present') if cron.nil?
|
79
|
+
end
|
80
|
+
|
81
|
+
def validate_job_class_related(options)
|
82
|
+
# Only support known job engines
|
83
|
+
unless Que::Scheduler::ToEnqueue.valid_job_class?(job_class)
|
84
|
+
err_field(:job_class, options, "Job #{job_class} was not a supported job type")
|
85
|
+
end
|
86
|
+
|
87
|
+
# queue name is only supported for a subrange of ActiveJob versions. Print this out as a
|
88
|
+
# warning.
|
89
|
+
if queue &&
|
90
|
+
Que::Scheduler::ToEnqueue.active_job_sufficient_version? &&
|
91
|
+
job_class < ::ActiveJob::Base &&
|
92
|
+
Que::Scheduler::ToEnqueue.active_job_version < Gem::Version.create('6.0.3')
|
93
|
+
puts <<-ERR
|
94
|
+
WARNING from que-scheduler....
|
95
|
+
Between versions 4.2.3 and 6.0.2 (inclusive) Rails did not support setting queue names
|
96
|
+
on que jobs with ActiveJob, so que-scheduler cannot support it.
|
97
|
+
See removed in Rails 4.2.3
|
98
|
+
https://github.com/rails/rails/pull/19498
|
99
|
+
And readded in Rails 6.0.3
|
100
|
+
https://github.com/rails/rails/pull/38635
|
101
|
+
|
102
|
+
Please remove all "queue" keys from ActiveJobs defined in the que-scheduler.yml config.
|
103
|
+
Specifically #{queue} for job #{name}.
|
104
|
+
ERR
|
105
|
+
end
|
106
|
+
end
|
62
107
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
108
|
+
def err_field(field, options, reason = '')
|
109
|
+
schedule = Que::Scheduler.configuration.schedule_location
|
110
|
+
value = options[field]
|
111
|
+
raise "Invalid #{field} '#{value}' for '#{name}' in que-scheduler schedule #{schedule}.\n" \
|
112
|
+
"#{reason}"
|
68
113
|
end
|
69
114
|
|
70
115
|
def generate_to_enqueue_list(missed_times)
|
@@ -75,10 +120,10 @@ module Que
|
|
75
120
|
|
76
121
|
if schedule_type == DefinedJob::DEFINED_JOB_TYPE_EVERY_EVENT
|
77
122
|
missed_times.map do |time_missed|
|
78
|
-
ToEnqueue.
|
123
|
+
ToEnqueue.create(options.merge(args: [time_missed] + args_array))
|
79
124
|
end
|
80
125
|
else
|
81
|
-
[ToEnqueue.
|
126
|
+
[ToEnqueue.create(options.merge(args: args_array))]
|
82
127
|
end
|
83
128
|
end
|
84
129
|
end
|
@@ -27,17 +27,15 @@ module Que
|
|
27
27
|
private
|
28
28
|
|
29
29
|
def hash_item_to_defined_job(name, defined_job_hash)
|
30
|
-
Que::Scheduler::DefinedJob.
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
}.compact
|
40
|
-
).freeze
|
30
|
+
Que::Scheduler::DefinedJob.create(
|
31
|
+
name: name,
|
32
|
+
job_class: defined_job_hash['class'] || name,
|
33
|
+
queue: defined_job_hash['queue'],
|
34
|
+
args: defined_job_hash['args'],
|
35
|
+
priority: defined_job_hash['priority'],
|
36
|
+
cron: defined_job_hash['cron'],
|
37
|
+
schedule_type: defined_job_hash['schedule_type']&.to_sym
|
38
|
+
)
|
41
39
|
end
|
42
40
|
end
|
43
41
|
end
|
@@ -4,6 +4,7 @@ require_relative 'schedule'
|
|
4
4
|
require_relative 'enqueueing_calculator'
|
5
5
|
require_relative 'scheduler_job_args'
|
6
6
|
require_relative 'state_checks'
|
7
|
+
require_relative 'to_enqueue'
|
7
8
|
require_relative 'version_support'
|
8
9
|
|
9
10
|
# The main job that runs every minute, determining what needs to be enqueued, enqueues the required
|
@@ -22,13 +23,11 @@ module Que
|
|
22
23
|
|
23
24
|
scheduler_job_args = SchedulerJobArgs.build(options)
|
24
25
|
logs = ["que-scheduler last ran at #{scheduler_job_args.last_run_time}."]
|
25
|
-
|
26
26
|
result = EnqueueingCalculator.parse(Scheduler.schedule.values, scheduler_job_args)
|
27
27
|
enqueued_jobs = enqueue_required_jobs(result, logs)
|
28
28
|
enqueue_self_again(
|
29
29
|
scheduler_job_args, scheduler_job_args.as_time, result.job_dictionary, enqueued_jobs
|
30
30
|
)
|
31
|
-
|
32
31
|
# Only now we're sure nothing errored, log the results
|
33
32
|
logs.each { |str| ::Que.log(event: 'que-scheduler'.to_sym, message: str) }
|
34
33
|
destroy
|
@@ -37,31 +36,23 @@ module Que
|
|
37
36
|
|
38
37
|
def enqueue_required_jobs(result, logs)
|
39
38
|
result.missed_jobs.map do |to_enqueue|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
enqueued_job =
|
44
|
-
if args.is_a?(Hash)
|
45
|
-
job_class.enqueue(args.merge(remaining_hash))
|
46
|
-
else
|
47
|
-
job_class.enqueue(*args, remaining_hash)
|
48
|
-
end
|
49
|
-
check_enqueued_job(enqueued_job, job_class, args, logs)
|
39
|
+
to_enqueue.enqueue.tap do |enqueued_job|
|
40
|
+
check_enqueued_job(to_enqueue, enqueued_job, logs)
|
41
|
+
end
|
50
42
|
end.compact
|
51
43
|
end
|
52
44
|
|
53
45
|
private
|
54
46
|
|
55
|
-
def check_enqueued_job(
|
56
|
-
if enqueued_job.
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
end
|
47
|
+
def check_enqueued_job(to_enqueue, enqueued_job, logs)
|
48
|
+
logs << if enqueued_job.present?
|
49
|
+
"que-scheduler enqueueing #{enqueued_job.job_class} " \
|
50
|
+
"#{enqueued_job.job_id} with args: #{enqueued_job.args}"
|
51
|
+
else
|
52
|
+
# This can happen if a middleware nixes the enqueue call
|
53
|
+
"que-scheduler called enqueue on #{to_enqueue.job_class} " \
|
54
|
+
'but it reported no job was scheduled. Has `enqueue` been overridden?'
|
55
|
+
end
|
65
56
|
end
|
66
57
|
|
67
58
|
def enqueue_self_again(scheduler_job_args, last_full_execution, job_dictionary, enqueued_jobs)
|
@@ -79,7 +70,7 @@ module Que
|
|
79
70
|
)
|
80
71
|
|
81
72
|
# rubocop:disable Style/GuardClause This reads better as a conditional
|
82
|
-
unless
|
73
|
+
unless Que::Scheduler::VersionSupport.job_attributes(enqueued_job).fetch(:job_id)
|
83
74
|
raise 'SchedulerJob could not self-schedule. Has `.enqueue` been monkey patched?'
|
84
75
|
end
|
85
76
|
# rubocop:enable Style/GuardClause
|
@@ -0,0 +1,150 @@
|
|
1
|
+
require 'que'
|
2
|
+
|
3
|
+
# This module uses polymorphic dispatch to centralise the differences between supporting Que::Job
|
4
|
+
# and other job systems.
|
5
|
+
module Que
|
6
|
+
module Scheduler
|
7
|
+
class ToEnqueue < Hashie::Dash
|
8
|
+
property :args, required: true, default: []
|
9
|
+
property :queue
|
10
|
+
property :priority
|
11
|
+
property :run_at, required: true
|
12
|
+
property :job_class, required: true
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def create(options)
|
16
|
+
type_from_job_class(options.fetch(:job_class)).new(
|
17
|
+
options.merge(run_at: Que::Scheduler::Db.now)
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def valid_job_class?(job_class)
|
22
|
+
type_from_job_class(job_class).present?
|
23
|
+
end
|
24
|
+
|
25
|
+
def active_job_version
|
26
|
+
Gem.loaded_specs['activejob']&.version
|
27
|
+
end
|
28
|
+
|
29
|
+
def active_job_sufficient_version?
|
30
|
+
# ActiveJob 4.x does not support job_ids correctly
|
31
|
+
# https://github.com/rails/rails/pull/20056/files
|
32
|
+
active_job_version && active_job_version > Gem::Version.create('5')
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def type_from_job_class(job_class)
|
38
|
+
types.each do |type, implementation|
|
39
|
+
return implementation if job_class < type
|
40
|
+
end
|
41
|
+
nil
|
42
|
+
end
|
43
|
+
|
44
|
+
def types
|
45
|
+
@types ||=
|
46
|
+
begin
|
47
|
+
hash = {
|
48
|
+
::Que::Job => QueJobType,
|
49
|
+
}
|
50
|
+
hash[::ActiveJob::Base] = ActiveJobType if ToEnqueue.active_job_sufficient_version?
|
51
|
+
hash
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# For jobs of type Que::Job
|
58
|
+
class QueJobType < ToEnqueue
|
59
|
+
def enqueue
|
60
|
+
job_settings = to_h.slice(:queue, :priority, :run_at).compact
|
61
|
+
job =
|
62
|
+
if args.is_a?(Hash)
|
63
|
+
job_class.enqueue(**args.merge(job_settings))
|
64
|
+
else
|
65
|
+
job_class.enqueue(*args, **job_settings)
|
66
|
+
end
|
67
|
+
|
68
|
+
return nil if job.nil? || !job # nil in Rails < 6.1, false after.
|
69
|
+
|
70
|
+
# Now read the just inserted job back out of the DB to get the actual values that will
|
71
|
+
# be used when the job is worked.
|
72
|
+
values = Que::Scheduler::VersionSupport.job_attributes(job).slice(
|
73
|
+
:args, :queue, :priority, :run_at, :job_class, :job_id
|
74
|
+
)
|
75
|
+
EnqueuedJobType.new(values)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# For jobs of type ActiveJob
|
80
|
+
class ActiveJobType < ToEnqueue
|
81
|
+
def enqueue
|
82
|
+
job = enqueue_active_job
|
83
|
+
|
84
|
+
return nil if job.nil? || !job # nil in Rails < 6.1, false after.
|
85
|
+
|
86
|
+
enqueued_values = calculate_enqueued_values(job)
|
87
|
+
EnqueuedJobType.new(enqueued_values)
|
88
|
+
end
|
89
|
+
|
90
|
+
private
|
91
|
+
|
92
|
+
def calculate_enqueued_values(job)
|
93
|
+
# Now read the just inserted job back out of the DB to get the actual values that will
|
94
|
+
# be used when the job is worked.
|
95
|
+
data = JSON.parse(job.to_json, symbolize_names: true)
|
96
|
+
|
97
|
+
# ActiveJob scheduled_at is returned as a float, where we want a Time for consistency
|
98
|
+
scheduled_at =
|
99
|
+
begin
|
100
|
+
scheduled_at_float = data[:scheduled_at]
|
101
|
+
scheduled_at_float ? Time.zone.at(scheduled_at_float) : nil
|
102
|
+
end
|
103
|
+
|
104
|
+
# Rails doesn't support queues for ActiveJob
|
105
|
+
# https://github.com/rails/rails/pull/19498
|
106
|
+
used_queue = nil
|
107
|
+
|
108
|
+
# We can't get the priority out of the DB, as the returned `job` doesn't give us access
|
109
|
+
# to the underlying ActiveJob that was scheduled. We have no option but to assume
|
110
|
+
# it was what we told it to use. If no priority was specified, we must assume it was
|
111
|
+
# the Que default, which is 100 t.ly/1jRK5
|
112
|
+
assume_used_priority = priority.nil? ? 100 : priority
|
113
|
+
|
114
|
+
{
|
115
|
+
args: data.fetch(:arguments),
|
116
|
+
queue: used_queue,
|
117
|
+
priority: assume_used_priority,
|
118
|
+
run_at: scheduled_at,
|
119
|
+
job_class: job_class.to_s,
|
120
|
+
job_id: data.fetch(:provider_job_id),
|
121
|
+
}
|
122
|
+
end
|
123
|
+
|
124
|
+
def enqueue_active_job
|
125
|
+
job_settings = {
|
126
|
+
priority: priority,
|
127
|
+
wait_until: run_at,
|
128
|
+
}.compact
|
129
|
+
|
130
|
+
job_class_set = job_class.set(**job_settings)
|
131
|
+
if args.is_a?(Hash)
|
132
|
+
job_class_set.perform_later(**args)
|
133
|
+
else
|
134
|
+
job_class_set.perform_later(*args)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
# A value object returned after a job has been enqueued. This is necessary as Que (normal) and
|
140
|
+
# ActiveJob return very different objects from the `enqueue` call.
|
141
|
+
class EnqueuedJobType < Hashie::Dash
|
142
|
+
property :args
|
143
|
+
property :queue
|
144
|
+
property :priority
|
145
|
+
property :run_at, required: true
|
146
|
+
property :job_class, required: true
|
147
|
+
property :job_id, required: true
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que-scheduler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Harry Lascelles
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-04-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -228,16 +228,16 @@ dependencies:
|
|
228
228
|
name: reek
|
229
229
|
requirement: !ruby/object:Gem::Requirement
|
230
230
|
requirements:
|
231
|
-
- -
|
231
|
+
- - ">="
|
232
232
|
- !ruby/object:Gem::Version
|
233
|
-
version:
|
233
|
+
version: '0'
|
234
234
|
type: :development
|
235
235
|
prerelease: false
|
236
236
|
version_requirements: !ruby/object:Gem::Requirement
|
237
237
|
requirements:
|
238
|
-
- -
|
238
|
+
- - ">="
|
239
239
|
- !ruby/object:Gem::Version
|
240
|
-
version:
|
240
|
+
version: '0'
|
241
241
|
- !ruby/object:Gem::Dependency
|
242
242
|
name: rspec
|
243
243
|
requirement: !ruby/object:Gem::Requirement
|
@@ -256,16 +256,16 @@ dependencies:
|
|
256
256
|
name: rubocop
|
257
257
|
requirement: !ruby/object:Gem::Requirement
|
258
258
|
requirements:
|
259
|
-
- -
|
259
|
+
- - '='
|
260
260
|
- !ruby/object:Gem::Version
|
261
|
-
version: 0.
|
261
|
+
version: 0.80.0
|
262
262
|
type: :development
|
263
263
|
prerelease: false
|
264
264
|
version_requirements: !ruby/object:Gem::Requirement
|
265
265
|
requirements:
|
266
|
-
- -
|
266
|
+
- - '='
|
267
267
|
- !ruby/object:Gem::Version
|
268
|
-
version: 0.
|
268
|
+
version: 0.80.0
|
269
269
|
- !ruby/object:Gem::Dependency
|
270
270
|
name: sqlite3
|
271
271
|
requirement: !ruby/object:Gem::Requirement
|
@@ -336,6 +336,7 @@ files:
|
|
336
336
|
- lib/que/scheduler/scheduler_job.rb
|
337
337
|
- lib/que/scheduler/scheduler_job_args.rb
|
338
338
|
- lib/que/scheduler/state_checks.rb
|
339
|
+
- lib/que/scheduler/to_enqueue.rb
|
339
340
|
- lib/que/scheduler/version.rb
|
340
341
|
- lib/que/scheduler/version_support.rb
|
341
342
|
homepage: https://github.com/hlascelles/que-scheduler
|
@@ -362,8 +363,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
362
363
|
- !ruby/object:Gem::Version
|
363
364
|
version: '0'
|
364
365
|
requirements: []
|
365
|
-
|
366
|
-
rubygems_version: 2.7.6.2
|
366
|
+
rubygems_version: 3.0.3
|
367
367
|
signing_key:
|
368
368
|
specification_version: 4
|
369
369
|
summary: A cron scheduler for Que
|