que-scheduler 3.2.8 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c2a35cace72621c452055873ea88a13aba5d00a63933df91b2d4d60840f11337
4
- data.tar.gz: 1b2d3bbb599260b37f8ca8c10a12bd03992b08ed74b0c75f47905b1f8238dfd5
3
+ metadata.gz: 5707036df23117e995d696388d17bfc0488eef9c7ba30211b1a3700e6f0fb94f
4
+ data.tar.gz: 3133e78529f3658c9509e0a1240189424caefc8219b7da035530e68fdbfb38c6
5
5
  SHA512:
6
- metadata.gz: ab5ad0c46554887d523d7b9f90c8336da989489e76cc69a0bfe5465b66b34e7a7c3d0f2fc6c0dc026660926b51e4fff0fde029600b93e52386147565a4c10a2a
7
- data.tar.gz: 7359bfdc4db605aea48c8bb87b1f02a8203dc103fa6e6c9ba5d6f33d051026c7e4abc7d4e5a503660d522d99f89c720455879a7b631929bd4e591961388afa75
6
+ metadata.gz: 4609167d5adfc2e7b9c583d3cbece89dd34e8624512b0c8c257804c05934964a343336b192944ce2790fde4f4b6029173df2f76d28fe7daee72e0a1ae473bad2
7
+ data.tar.gz: d55658a7923c50010a8a38713dc25c0f08d182004cb5156a336921f55b35e91631edb042f5e76f194d58ed04245ea8f08c78dd64386c2479ee4bdb8ba9a4512a
data/README.md CHANGED
@@ -235,3 +235,4 @@ This gem was inspired by the makers of the excellent [Que](https://github.com/ch
235
235
  * @jish
236
236
  * @joehorsnell
237
237
  * @bnauta
238
+ * @papodaca
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'to_enqueue'
4
+
3
5
  module Que
4
6
  module Scheduler
5
7
  module Audit
@@ -24,11 +26,10 @@ module Que
24
26
  def append(scheduler_job_id, executed_at, enqueued_jobs)
25
27
  ::Que::Scheduler::VersionSupport.execute(INSERT_AUDIT, [scheduler_job_id, executed_at])
26
28
  enqueued_jobs.each do |j|
27
- attrs = Que::Scheduler::VersionSupport.job_attributes(j)
28
29
  inserted = ::Que::Scheduler::VersionSupport.execute(
29
30
  INSERT_AUDIT_ENQUEUED,
30
31
  [scheduler_job_id] +
31
- attrs.values_at(:job_class, :queue, :priority, :args, :job_id, :run_at)
32
+ j.values_at(:job_class, :queue, :priority, :args, :job_id, :run_at)
32
33
  )
33
34
  raise "Cannot save audit row #{scheduler_job_id} #{executed_at} #{j}" if inserted.empty?
34
35
  end
@@ -10,23 +10,23 @@ module Que
10
10
 
11
11
  DEFINED_JOB_TYPES = [
12
12
  DEFINED_JOB_TYPE_DEFAULT = :default,
13
- DEFINED_JOB_TYPE_EVERY_EVENT = :every_event
13
+ DEFINED_JOB_TYPE_EVERY_EVENT = :every_event,
14
14
  ].freeze
15
15
 
16
- property :name, required: true
17
- property :job_class, required: true, transform_with: lambda { |v|
18
- job_class = Object.const_get(v)
19
- job_class < Que::Job ? job_class : err_field(:job_class, v)
20
- }
21
- property :cron, required: true, transform_with: lambda { |v|
22
- Fugit::Cron.parse(v) || err_field(:cron, v)
23
- }
24
- property :queue, transform_with: ->(v) { v.is_a?(String) ? v : err_field(:queue, v) }
25
- property :priority, transform_with: ->(v) { v.is_a?(Integer) ? v : err_field(:priority, v) }
16
+ property :name
17
+ property :job_class, transform_with: ->(v) { Object.const_get(v) }
18
+ property :cron, transform_with: ->(v) { Fugit::Cron.parse(v) }
19
+ property :queue
20
+ property :priority
26
21
  property :args
27
- property :schedule_type, default: DEFINED_JOB_TYPE_DEFAULT, transform_with: lambda { |v|
28
- v.to_sym.tap { |vs| DEFINED_JOB_TYPES.include?(vs) || err_field(:schedule_type, v) }
29
- }
22
+ property :schedule_type, default: DEFINED_JOB_TYPE_DEFAULT
23
+
24
+ class << self
25
+ def create(options)
26
+ defined_job = new(options.compact)
27
+ defined_job.freeze.tap { |dj| dj.validate(options) }
28
+ end
29
+ end
30
30
 
31
31
  # Given a "last time", return the next Time the event will occur, or nil if it
32
32
  # is after "to".
@@ -49,22 +49,67 @@ module Que
49
49
  generate_to_enqueue_list(missed_times)
50
50
  end
51
51
 
52
- class << self
53
- private
52
+ def validate(options)
53
+ validate_fields_presence(options)
54
+ validate_fields_types(options)
55
+ validate_job_class_related(options)
56
+ end
57
+
58
+ private
54
59
 
55
- def err_field(field, value)
56
- schedule = Que::Scheduler.configuration.schedule_location
57
- raise "Invalid #{field} '#{value}' in que-scheduler schedule #{schedule}"
60
+ # rubocop:disable Style/GuardClause This reads better as a conditional
61
+ def validate_fields_types(options)
62
+ unless queue.nil? || queue.is_a?(String)
63
+ err_field(:queue, options, 'queue must be a string')
64
+ end
65
+ unless priority.nil? || priority.is_a?(Integer)
66
+ err_field(:priority, options, 'priority must be an integer')
67
+ end
68
+ unless DEFINED_JOB_TYPES.include?(schedule_type)
69
+ err_field(:schedule_type, options, "Not in #{DEFINED_JOB_TYPES}")
58
70
  end
59
71
  end
72
+ # rubocop:enable Style/GuardClause
60
73
 
61
- private
74
+ def validate_fields_presence(options)
75
+ err_field(:name, options, 'name must be present') if name.nil?
76
+ err_field(:job_class, options, 'job_class must be present') if job_class.nil?
77
+ # An invalid cron is nil
78
+ err_field(:cron, options, 'cron must be present') if cron.nil?
79
+ end
80
+
81
+ def validate_job_class_related(options)
82
+ # Only support known job engines
83
+ unless Que::Scheduler::ToEnqueue.valid_job_class?(job_class)
84
+ err_field(:job_class, options, "Job #{job_class} was not a supported job type")
85
+ end
86
+
87
+ # queue name is only supported for a subrange of ActiveJob versions. Print this out as a
88
+ # warning.
89
+ if queue &&
90
+ Que::Scheduler::ToEnqueue.active_job_sufficient_version? &&
91
+ job_class < ::ActiveJob::Base &&
92
+ Que::Scheduler::ToEnqueue.active_job_version < Gem::Version.create('6.0.3')
93
+ puts <<-ERR
94
+ WARNING from que-scheduler....
95
+ Between versions 4.2.3 and 6.0.2 (inclusive) Rails did not support setting queue names
96
+ on que jobs with ActiveJob, so que-scheduler cannot support it.
97
+ See removed in Rails 4.2.3
98
+ https://github.com/rails/rails/pull/19498
99
+ And readded in Rails 6.0.3
100
+ https://github.com/rails/rails/pull/38635
101
+
102
+ Please remove all "queue" keys from ActiveJobs defined in the que-scheduler.yml config.
103
+ Specifically #{queue} for job #{name}.
104
+ ERR
105
+ end
106
+ end
62
107
 
63
- class ToEnqueue < Hashie::Dash
64
- property :args, required: true, default: []
65
- property :queue
66
- property :priority
67
- property :job_class, required: true
108
+ def err_field(field, options, reason = '')
109
+ schedule = Que::Scheduler.configuration.schedule_location
110
+ value = options[field]
111
+ raise "Invalid #{field} '#{value}' for '#{name}' in que-scheduler schedule #{schedule}.\n" \
112
+ "#{reason}"
68
113
  end
69
114
 
70
115
  def generate_to_enqueue_list(missed_times)
@@ -75,10 +120,10 @@ module Que
75
120
 
76
121
  if schedule_type == DefinedJob::DEFINED_JOB_TYPE_EVERY_EVENT
77
122
  missed_times.map do |time_missed|
78
- ToEnqueue.new(options.merge(args: [time_missed] + args_array))
123
+ ToEnqueue.create(options.merge(args: [time_missed] + args_array))
79
124
  end
80
125
  else
81
- [ToEnqueue.new(options.merge(args: args_array))]
126
+ [ToEnqueue.create(options.merge(args: args_array))]
82
127
  end
83
128
  end
84
129
  end
@@ -27,17 +27,15 @@ module Que
27
27
  private
28
28
 
29
29
  def hash_item_to_defined_job(name, defined_job_hash)
30
- Que::Scheduler::DefinedJob.new(
31
- {
32
- name: name,
33
- job_class: defined_job_hash['class'] || name,
34
- queue: defined_job_hash['queue'],
35
- args: defined_job_hash['args'],
36
- priority: defined_job_hash['priority'],
37
- cron: defined_job_hash['cron'],
38
- schedule_type: defined_job_hash['schedule_type'],
39
- }.compact
40
- ).freeze
30
+ Que::Scheduler::DefinedJob.create(
31
+ name: name,
32
+ job_class: defined_job_hash['class'] || name,
33
+ queue: defined_job_hash['queue'],
34
+ args: defined_job_hash['args'],
35
+ priority: defined_job_hash['priority'],
36
+ cron: defined_job_hash['cron'],
37
+ schedule_type: defined_job_hash['schedule_type']&.to_sym
38
+ )
41
39
  end
42
40
  end
43
41
  end
@@ -4,6 +4,7 @@ require_relative 'schedule'
4
4
  require_relative 'enqueueing_calculator'
5
5
  require_relative 'scheduler_job_args'
6
6
  require_relative 'state_checks'
7
+ require_relative 'to_enqueue'
7
8
  require_relative 'version_support'
8
9
 
9
10
  # The main job that runs every minute, determining what needs to be enqueued, enqueues the required
@@ -22,13 +23,11 @@ module Que
22
23
 
23
24
  scheduler_job_args = SchedulerJobArgs.build(options)
24
25
  logs = ["que-scheduler last ran at #{scheduler_job_args.last_run_time}."]
25
-
26
26
  result = EnqueueingCalculator.parse(Scheduler.schedule.values, scheduler_job_args)
27
27
  enqueued_jobs = enqueue_required_jobs(result, logs)
28
28
  enqueue_self_again(
29
29
  scheduler_job_args, scheduler_job_args.as_time, result.job_dictionary, enqueued_jobs
30
30
  )
31
-
32
31
  # Only now we're sure nothing errored, log the results
33
32
  logs.each { |str| ::Que.log(event: 'que-scheduler'.to_sym, message: str) }
34
33
  destroy
@@ -37,31 +36,23 @@ module Que
37
36
 
38
37
  def enqueue_required_jobs(result, logs)
39
38
  result.missed_jobs.map do |to_enqueue|
40
- job_class = to_enqueue.job_class
41
- args = to_enqueue.args
42
- remaining_hash = to_enqueue.except(:job_class, :args)
43
- enqueued_job =
44
- if args.is_a?(Hash)
45
- job_class.enqueue(args.merge(remaining_hash))
46
- else
47
- job_class.enqueue(*args, remaining_hash)
48
- end
49
- check_enqueued_job(enqueued_job, job_class, args, logs)
39
+ to_enqueue.enqueue.tap do |enqueued_job|
40
+ check_enqueued_job(to_enqueue, enqueued_job, logs)
41
+ end
50
42
  end.compact
51
43
  end
52
44
 
53
45
  private
54
46
 
55
- def check_enqueued_job(enqueued_job, job_class, args, logs)
56
- if enqueued_job.is_a?(Que::Job)
57
- job_id = Que::Scheduler::VersionSupport.job_attributes(enqueued_job).fetch(:job_id)
58
- logs << "que-scheduler enqueueing #{job_class} #{job_id} with args: #{args}"
59
- enqueued_job
60
- else
61
- # This can happen if a middleware nixes the enqueue call
62
- logs << "que-scheduler called enqueue on #{job_class} but did not receive a #{Que::Job}"
63
- nil
64
- end
47
+ def check_enqueued_job(to_enqueue, enqueued_job, logs)
48
+ logs << if enqueued_job.present?
49
+ "que-scheduler enqueueing #{enqueued_job.job_class} " \
50
+ "#{enqueued_job.job_id} with args: #{enqueued_job.args}"
51
+ else
52
+ # This can happen if a middleware nixes the enqueue call
53
+ "que-scheduler called enqueue on #{to_enqueue.job_class} " \
54
+ 'but it reported no job was scheduled. Has `enqueue` been overridden?'
55
+ end
65
56
  end
66
57
 
67
58
  def enqueue_self_again(scheduler_job_args, last_full_execution, job_dictionary, enqueued_jobs)
@@ -79,7 +70,7 @@ module Que
79
70
  )
80
71
 
81
72
  # rubocop:disable Style/GuardClause This reads better as a conditional
82
- unless check_enqueued_job(enqueued_job, SchedulerJob, {}, [])
73
+ unless Que::Scheduler::VersionSupport.job_attributes(enqueued_job).fetch(:job_id)
83
74
  raise 'SchedulerJob could not self-schedule. Has `.enqueue` been monkey patched?'
84
75
  end
85
76
  # rubocop:enable Style/GuardClause
@@ -0,0 +1,150 @@
1
+ require 'que'
2
+
3
+ # This module uses polymorphic dispatch to centralise the differences between supporting Que::Job
4
+ # and other job systems.
5
+ module Que
6
+ module Scheduler
7
+ class ToEnqueue < Hashie::Dash
8
+ property :args, required: true, default: []
9
+ property :queue
10
+ property :priority
11
+ property :run_at, required: true
12
+ property :job_class, required: true
13
+
14
+ class << self
15
+ def create(options)
16
+ type_from_job_class(options.fetch(:job_class)).new(
17
+ options.merge(run_at: Que::Scheduler::Db.now)
18
+ )
19
+ end
20
+
21
+ def valid_job_class?(job_class)
22
+ type_from_job_class(job_class).present?
23
+ end
24
+
25
+ def active_job_version
26
+ Gem.loaded_specs['activejob']&.version
27
+ end
28
+
29
+ def active_job_sufficient_version?
30
+ # ActiveJob 4.x does not support job_ids correctly
31
+ # https://github.com/rails/rails/pull/20056/files
32
+ active_job_version && active_job_version > Gem::Version.create('5')
33
+ end
34
+
35
+ private
36
+
37
+ def type_from_job_class(job_class)
38
+ types.each do |type, implementation|
39
+ return implementation if job_class < type
40
+ end
41
+ nil
42
+ end
43
+
44
+ def types
45
+ @types ||=
46
+ begin
47
+ hash = {
48
+ ::Que::Job => QueJobType,
49
+ }
50
+ hash[::ActiveJob::Base] = ActiveJobType if ToEnqueue.active_job_sufficient_version?
51
+ hash
52
+ end
53
+ end
54
+ end
55
+ end
56
+
57
+ # For jobs of type Que::Job
58
+ class QueJobType < ToEnqueue
59
+ def enqueue
60
+ job_settings = to_h.slice(:queue, :priority, :run_at).compact
61
+ job =
62
+ if args.is_a?(Hash)
63
+ job_class.enqueue(**args.merge(job_settings))
64
+ else
65
+ job_class.enqueue(*args, **job_settings)
66
+ end
67
+
68
+ return nil if job.nil? || !job # nil in Rails < 6.1, false after.
69
+
70
+ # Now read the just inserted job back out of the DB to get the actual values that will
71
+ # be used when the job is worked.
72
+ values = Que::Scheduler::VersionSupport.job_attributes(job).slice(
73
+ :args, :queue, :priority, :run_at, :job_class, :job_id
74
+ )
75
+ EnqueuedJobType.new(values)
76
+ end
77
+ end
78
+
79
+ # For jobs of type ActiveJob
80
+ class ActiveJobType < ToEnqueue
81
+ def enqueue
82
+ job = enqueue_active_job
83
+
84
+ return nil if job.nil? || !job # nil in Rails < 6.1, false after.
85
+
86
+ enqueued_values = calculate_enqueued_values(job)
87
+ EnqueuedJobType.new(enqueued_values)
88
+ end
89
+
90
+ private
91
+
92
+ def calculate_enqueued_values(job)
93
+ # Now read the just inserted job back out of the DB to get the actual values that will
94
+ # be used when the job is worked.
95
+ data = JSON.parse(job.to_json, symbolize_names: true)
96
+
97
+ # ActiveJob scheduled_at is returned as a float, where we want a Time for consistency
98
+ scheduled_at =
99
+ begin
100
+ scheduled_at_float = data[:scheduled_at]
101
+ scheduled_at_float ? Time.zone.at(scheduled_at_float) : nil
102
+ end
103
+
104
+ # Rails doesn't support queues for ActiveJob
105
+ # https://github.com/rails/rails/pull/19498
106
+ used_queue = nil
107
+
108
+ # We can't get the priority out of the DB, as the returned `job` doesn't give us access
109
+ # to the underlying ActiveJob that was scheduled. We have no option but to assume
110
+ # it was what we told it to use. If no priority was specified, we must assume it was
111
+ # the Que default, which is 100 t.ly/1jRK5
112
+ assume_used_priority = priority.nil? ? 100 : priority
113
+
114
+ {
115
+ args: data.fetch(:arguments),
116
+ queue: used_queue,
117
+ priority: assume_used_priority,
118
+ run_at: scheduled_at,
119
+ job_class: job_class.to_s,
120
+ job_id: data.fetch(:provider_job_id),
121
+ }
122
+ end
123
+
124
+ def enqueue_active_job
125
+ job_settings = {
126
+ priority: priority,
127
+ wait_until: run_at,
128
+ }.compact
129
+
130
+ job_class_set = job_class.set(**job_settings)
131
+ if args.is_a?(Hash)
132
+ job_class_set.perform_later(**args)
133
+ else
134
+ job_class_set.perform_later(*args)
135
+ end
136
+ end
137
+ end
138
+
139
+ # A value object returned after a job has been enqueued. This is necessary as Que (normal) and
140
+ # ActiveJob return very different objects from the `enqueue` call.
141
+ class EnqueuedJobType < Hashie::Dash
142
+ property :args
143
+ property :queue
144
+ property :priority
145
+ property :run_at, required: true
146
+ property :job_class, required: true
147
+ property :job_id, required: true
148
+ end
149
+ end
150
+ end
@@ -1,5 +1,5 @@
1
1
  module Que
2
2
  module Scheduler
3
- VERSION = '3.2.8'.freeze
3
+ VERSION = '3.3.0'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: que-scheduler
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.2.8
4
+ version: 3.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Harry Lascelles
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-09 00:00:00.000000000 Z
11
+ date: 2020-04-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -228,16 +228,16 @@ dependencies:
228
228
  name: reek
229
229
  requirement: !ruby/object:Gem::Requirement
230
230
  requirements:
231
- - - '='
231
+ - - ">="
232
232
  - !ruby/object:Gem::Version
233
- version: 5.0.2
233
+ version: '0'
234
234
  type: :development
235
235
  prerelease: false
236
236
  version_requirements: !ruby/object:Gem::Requirement
237
237
  requirements:
238
- - - '='
238
+ - - ">="
239
239
  - !ruby/object:Gem::Version
240
- version: 5.0.2
240
+ version: '0'
241
241
  - !ruby/object:Gem::Dependency
242
242
  name: rspec
243
243
  requirement: !ruby/object:Gem::Requirement
@@ -256,16 +256,16 @@ dependencies:
256
256
  name: rubocop
257
257
  requirement: !ruby/object:Gem::Requirement
258
258
  requirements:
259
- - - "~>"
259
+ - - '='
260
260
  - !ruby/object:Gem::Version
261
- version: 0.68.1
261
+ version: 0.80.0
262
262
  type: :development
263
263
  prerelease: false
264
264
  version_requirements: !ruby/object:Gem::Requirement
265
265
  requirements:
266
- - - "~>"
266
+ - - '='
267
267
  - !ruby/object:Gem::Version
268
- version: 0.68.1
268
+ version: 0.80.0
269
269
  - !ruby/object:Gem::Dependency
270
270
  name: sqlite3
271
271
  requirement: !ruby/object:Gem::Requirement
@@ -336,6 +336,7 @@ files:
336
336
  - lib/que/scheduler/scheduler_job.rb
337
337
  - lib/que/scheduler/scheduler_job_args.rb
338
338
  - lib/que/scheduler/state_checks.rb
339
+ - lib/que/scheduler/to_enqueue.rb
339
340
  - lib/que/scheduler/version.rb
340
341
  - lib/que/scheduler/version_support.rb
341
342
  homepage: https://github.com/hlascelles/que-scheduler
@@ -362,8 +363,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
362
363
  - !ruby/object:Gem::Version
363
364
  version: '0'
364
365
  requirements: []
365
- rubyforge_project:
366
- rubygems_version: 2.7.6.2
366
+ rubygems_version: 3.0.3
367
367
  signing_key:
368
368
  specification_version: 4
369
369
  summary: A cron scheduler for Que