que-scheduler 3.4.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +68 -40
- data/lib/que-scheduler.rb +1 -1
- data/lib/que/scheduler.rb +8 -8
- data/lib/que/scheduler/audit.rb +3 -3
- data/lib/que/scheduler/configuration.rb +35 -0
- data/lib/que/scheduler/db.rb +2 -2
- data/lib/que/scheduler/defined_job.rb +9 -9
- data/lib/que/scheduler/enqueueing_calculator.rb +1 -1
- data/lib/que/scheduler/jobs/que_scheduler_audit_clear_down_job.rb +4 -4
- data/lib/que/scheduler/migrations/6/down.sql +7 -0
- data/lib/que/scheduler/migrations/6/up.sql +26 -0
- data/lib/que/scheduler/schedule.rb +29 -12
- data/lib/que/scheduler/scheduler_job.rb +18 -15
- data/lib/que/scheduler/scheduler_job_args.rb +3 -3
- data/lib/que/scheduler/state_checks.rb +3 -19
- data/lib/que/scheduler/to_enqueue.rb +4 -4
- data/lib/que/scheduler/version.rb +1 -1
- data/lib/que/scheduler/version_support.rb +9 -9
- metadata +19 -3
- data/lib/que/scheduler/config.rb +0 -27
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 78c9bc8d137dbe8a7c2428e50c615928c5d702d06432763b5346030fcbe81d5b
|
4
|
+
data.tar.gz: 3e639fd3f2263266fa25e25c637390f718e7751726feaefaa63f69cc91124f2d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b599d23b38d8263412e2a1fa7eb6621f564a628ae7b66800b0259667a76956e0a9e16824bd463a24edd0752119f30ec01ac3a1951e1d4abfc1d05ed1905745dc
|
7
|
+
data.tar.gz: 42691328a2727e14f15305e6baac0f0d7b3e4cb1d1a91c80d2ef1f7ab42db56e3bd1640884805adf381d65fbe2d6b3d7abbe2373e8ef49dfbeb737f790b76700
|
data/README.md
CHANGED
@@ -18,9 +18,9 @@ needs to be run, enqueueing those jobs, then enqueueing itself to check again la
|
|
18
18
|
```ruby
|
19
19
|
gem 'que-scheduler'
|
20
20
|
```
|
21
|
-
1. Specify a schedule in a yml file (see below). The default location that
|
22
|
-
look for it is `config/que_schedule.yml`.
|
23
|
-
files, but with additional features.
|
21
|
+
1. Specify a schedule in a yml file or programmatically (see below). The default location that
|
22
|
+
que-scheduler will look for it is `config/que_schedule.yml`. The format is essentially the same as
|
23
|
+
resque-scheduler files, but with additional features.
|
24
24
|
|
25
25
|
1. Add a migration to start the job scheduler and prepare the audit table. Note that this migration
|
26
26
|
will fail if Que is set to execute jobs synchronously, i.e. `Que::Job.run_synchronously = true`.
|
@@ -28,14 +28,18 @@ files, but with additional features.
|
|
28
28
|
```ruby
|
29
29
|
class CreateQueSchedulerSchema < ActiveRecord::Migration
|
30
30
|
def change
|
31
|
-
Que::Scheduler::Migrations.migrate!(version:
|
31
|
+
Que::Scheduler::Migrations.migrate!(version: 6)
|
32
32
|
end
|
33
33
|
end
|
34
34
|
```
|
35
35
|
|
36
36
|
## Schedule configuration
|
37
37
|
|
38
|
-
The schedule file
|
38
|
+
The schedule file should be placed here: `config/que_schedule.yml`. Alternatively if you
|
39
|
+
wish to generate the configuration dynamically, you can set it directly using an initializer
|
40
|
+
(see "Gem configuration" below).
|
41
|
+
|
42
|
+
The file is a list of que job classes with arguments and a schedule frequency (in crontab
|
39
43
|
syntax). The format is similar to the resque-scheduler format, though priorities must be supplied as
|
40
44
|
integers, and job classes must be migrated from Resque to Que. Cron syntax can be anything
|
41
45
|
understood by [fugit](https://github.com/floraison/fugit#fugitcron).
|
@@ -131,19 +135,34 @@ A job can have a `schedule_type` assigned to it. Valid values are:
|
|
131
135
|
|
132
136
|
## Gem configuration
|
133
137
|
|
134
|
-
You can configure some aspects of the gem with
|
138
|
+
You can configure some aspects of the gem with a config block (eg in a Rails initializer).
|
139
|
+
The default is given below. You can omit any configuration sections you are not intending to change.
|
140
|
+
It is quite likely you won't have to create this config at all.
|
135
141
|
|
136
142
|
```ruby
|
137
143
|
Que::Scheduler.configure do |config|
|
138
144
|
# The location of the schedule yaml file.
|
139
|
-
config.schedule_location = ENV.fetch(
|
145
|
+
config.schedule_location = ENV.fetch("QUE_SCHEDULER_CONFIG_LOCATION", "config/que_schedule.yml")
|
146
|
+
|
147
|
+
# The schedule as a hash. You can use this if you want to build the schedule yourself at runtime.
|
148
|
+
# This will override the above value if provided.
|
149
|
+
config.schedule = {
|
150
|
+
SpecifiedByHashTestJob: {
|
151
|
+
cron: "02 11 * * *"
|
152
|
+
}
|
153
|
+
}
|
140
154
|
|
141
|
-
#
|
142
|
-
# However
|
155
|
+
# The transaction block adapter. By default, que-scheduler uses the one supplied by que.
|
156
|
+
# However if, for example, you rely on listeners to ActiveRecord's exact `transaction` method, or
|
143
157
|
# Sequel's DB.after_commit helper, then you can supply it here.
|
144
158
|
config.transaction_adapter = ::Que.method(:transaction)
|
145
|
-
end
|
146
159
|
|
160
|
+
# Which queue name the que-scheduler job should self-schedule on. Typically this is the default
|
161
|
+
# queue of que, which has a different name in Que 0.x ("") and 1.x ("default").
|
162
|
+
# It *must* be the "highest throughput" queue - do not work the scheduler on a "long
|
163
|
+
# running jobs" queue. It is very unlikely you will want to change this.
|
164
|
+
config.que_scheduler_queue = ENV.fetch("QUE_SCHEDULER_QUEUE", "" or "default")
|
165
|
+
end
|
147
166
|
```
|
148
167
|
|
149
168
|
## Scheduler Audit
|
@@ -155,20 +174,40 @@ migration tasks.
|
|
155
174
|
Additionally, there is the audit table `que_scheduler_audit_enqueued`. This logs every job that
|
156
175
|
the scheduler enqueues.
|
157
176
|
|
177
|
+
que-scheduler comes with the `QueSchedulerAuditClearDownJob` job built in that you can optionally
|
178
|
+
schedule to clear down audit rows if you don't need to retain them indefinitely. You should add this
|
179
|
+
to your own scheduler config yaml.
|
180
|
+
|
181
|
+
For example:
|
182
|
+
|
183
|
+
```yaml
|
184
|
+
# This will clear down the oldest que-scheduler audit rows. Since que-scheduler
|
185
|
+
# runs approximately every minute, 129600 is 90 days.
|
186
|
+
Que::Scheduler::Jobs::QueSchedulerAuditClearDownJob:
|
187
|
+
cron: "0 0 * * *"
|
188
|
+
args:
|
189
|
+
retain_row_count: 129600
|
190
|
+
```
|
191
|
+
|
192
|
+
## Required migrations
|
193
|
+
|
158
194
|
When there is a major version (breaking) change, a migration should be run in. The version of the
|
159
|
-
migration proceeds at a faster rate than the version of the gem.
|
160
|
-
|
195
|
+
latest migration proceeds at a faster rate than the version of the gem. eg If the gem is on version
|
196
|
+
3 then the migrations may be on version 6).
|
197
|
+
|
198
|
+
To run in all the migrations required up to a number, just migrate to that number with one line, and
|
199
|
+
it will perform all the intermediary steps.
|
161
200
|
|
162
201
|
ie, This will perform all migrations necessary up to the latest version, skipping any already
|
163
202
|
performed.
|
164
203
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
204
|
+
```ruby
|
205
|
+
class CreateQueSchedulerSchema < ActiveRecord::Migration
|
206
|
+
def change
|
207
|
+
Que::Scheduler::Migrations.migrate!(version: 6)
|
208
|
+
end
|
209
|
+
end
|
210
|
+
```
|
172
211
|
|
173
212
|
The changes in past migrations were:
|
174
213
|
|
@@ -179,23 +218,7 @@ The changes in past migrations were:
|
|
179
218
|
| 3 | Added the audit table `que_scheduler_audit_enqueued`. |
|
180
219
|
| 4 | Updated the the audit tables to use bigints |
|
181
220
|
| 5 | Dropped an unnecessary index |
|
182
|
-
|
183
|
-
## Built in optional job for audit clear down
|
184
|
-
|
185
|
-
que-scheduler comes with the `QueSchedulerAuditClearDownJob` job built in that you can optionally
|
186
|
-
schedule to clear down audit rows if you don't need to retain them indefinitely. You should add this
|
187
|
-
to your own scheduler config yaml.
|
188
|
-
|
189
|
-
For example:
|
190
|
-
|
191
|
-
```yaml
|
192
|
-
# This will clear down the oldest que-scheduler audit rows. Since que-scheduler
|
193
|
-
# runs approximately every minute, 129600 is 90 days.
|
194
|
-
Que::Scheduler::Jobs::QueSchedulerAuditClearDownJob:
|
195
|
-
cron: "0 0 * * *"
|
196
|
-
args:
|
197
|
-
retain_row_count: 129600
|
198
|
-
```
|
221
|
+
| 6 | Enforced single scheduler job at the trigger level |
|
199
222
|
|
200
223
|
## HA Redundancy and DB restores
|
201
224
|
|
@@ -210,8 +233,8 @@ in a coherent state with the rest of your database.
|
|
210
233
|
## Concurrent scheduler detection
|
211
234
|
|
212
235
|
No matter how many tasks you have defined in your schedule, you will only ever need one que-scheduler
|
213
|
-
job enqueued. que-scheduler knows this, and
|
214
|
-
|
236
|
+
job enqueued. que-scheduler knows this, and there are DB constraints in place to ensure there is
|
237
|
+
only ever exactly one scheduler job.
|
215
238
|
|
216
239
|
It also follows que job design [best practices](https://github.com/chanks/que/blob/master/docs/writing_reliable_jobs.md),
|
217
240
|
using ACID guarantees, to ensure that it will never run multiple times. If the scheduler crashes for any reason,
|
@@ -263,6 +286,10 @@ The scheduler will then continue to retry indefinitely.
|
|
263
286
|
que-scheduler uses [semantic versioning](https://semver.org/), so major version changes will usually
|
264
287
|
require additional actions to be taken upgrading from one major version to another.
|
265
288
|
|
289
|
+
## Changelog
|
290
|
+
|
291
|
+
A full changelog can be found here: [CHANGELOG.md](https://github.com/hlascelles/que-scheduler/blob/master/CHANGELOG.md)
|
292
|
+
|
266
293
|
## System requirements
|
267
294
|
|
268
295
|
Your [postgres](https://www.postgresql.org/) database must be at least version 9.4.0.
|
@@ -273,8 +300,9 @@ This gem was inspired by the makers of the excellent [Que](https://github.com/ch
|
|
273
300
|
|
274
301
|
## Contributors
|
275
302
|
|
303
|
+
* @bnauta
|
304
|
+
* @JackDanger
|
276
305
|
* @jish
|
277
306
|
* @joehorsnell
|
278
|
-
* @bnauta
|
279
|
-
* @papodaca
|
280
307
|
* @krzyzak
|
308
|
+
* @papodaca
|
data/lib/que-scheduler.rb
CHANGED
data/lib/que/scheduler.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
1
|
+
require "que/scheduler/version"
|
2
|
+
require "que/scheduler/version_support"
|
3
|
+
require "que/scheduler/configuration"
|
4
|
+
require "que/scheduler/scheduler_job"
|
5
|
+
require "que/scheduler/db"
|
6
|
+
require "que/scheduler/audit"
|
7
|
+
require "que/scheduler/migrations"
|
8
|
+
require "que/scheduler/jobs/que_scheduler_audit_clear_down_job"
|
data/lib/que/scheduler/audit.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative "to_enqueue"
|
4
4
|
|
5
5
|
module Que
|
6
6
|
module Scheduler
|
7
7
|
module Audit
|
8
|
-
TABLE_NAME =
|
9
|
-
ENQUEUED_TABLE_NAME =
|
8
|
+
TABLE_NAME = "que_scheduler_audit"
|
9
|
+
ENQUEUED_TABLE_NAME = "que_scheduler_audit_enqueued"
|
10
10
|
INSERT_AUDIT = %{
|
11
11
|
INSERT INTO #{TABLE_NAME} (scheduler_job_id, executed_at)
|
12
12
|
VALUES ($1::bigint, $2::timestamptz)
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require "que"
|
2
|
+
require_relative "version_support"
|
3
|
+
|
4
|
+
module Que
|
5
|
+
module Scheduler
|
6
|
+
class Configuration
|
7
|
+
attr_accessor :schedule_location
|
8
|
+
attr_accessor :schedule
|
9
|
+
attr_accessor :transaction_adapter
|
10
|
+
attr_accessor :que_scheduler_queue
|
11
|
+
end
|
12
|
+
|
13
|
+
class << self
|
14
|
+
attr_accessor :configuration
|
15
|
+
|
16
|
+
def configure
|
17
|
+
self.configuration ||= Configuration.new
|
18
|
+
yield(configuration)
|
19
|
+
end
|
20
|
+
|
21
|
+
def apply_defaults
|
22
|
+
configure do |config|
|
23
|
+
config.schedule_location =
|
24
|
+
ENV.fetch("QUE_SCHEDULER_CONFIG_LOCATION", "config/que_schedule.yml")
|
25
|
+
config.transaction_adapter = ::Que.method(:transaction)
|
26
|
+
config.que_scheduler_queue =
|
27
|
+
ENV.fetch("QUE_SCHEDULER_QUEUE", Que::Scheduler::VersionSupport.default_scheduler_queue)
|
28
|
+
config.schedule = nil
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
Que::Scheduler.apply_defaults
|
data/lib/que/scheduler/db.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative "configuration"
|
4
4
|
|
5
5
|
module Que
|
6
6
|
module Scheduler
|
7
7
|
module Db
|
8
8
|
SCHEDULER_COUNT_SQL =
|
9
9
|
"SELECT COUNT(*) FROM que_jobs WHERE job_class = 'Que::Scheduler::SchedulerJob'"
|
10
|
-
NOW_SQL =
|
10
|
+
NOW_SQL = "SELECT now()"
|
11
11
|
|
12
12
|
class << self
|
13
13
|
def count_schedulers
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "hashie"
|
2
|
+
require "fugit"
|
3
3
|
|
4
4
|
# This is the definition of one scheduleable job in the que-scheduler config yml file.
|
5
5
|
module Que
|
@@ -59,10 +59,10 @@ module Que
|
|
59
59
|
# rubocop:disable Style/GuardClause This reads better as a conditional
|
60
60
|
def validate_fields_types(options)
|
61
61
|
unless queue.nil? || queue.is_a?(String)
|
62
|
-
err_field(:queue, options,
|
62
|
+
err_field(:queue, options, "queue must be a string")
|
63
63
|
end
|
64
64
|
unless priority.nil? || priority.is_a?(Integer)
|
65
|
-
err_field(:priority, options,
|
65
|
+
err_field(:priority, options, "priority must be an integer")
|
66
66
|
end
|
67
67
|
unless DEFINED_JOB_TYPES.include?(schedule_type)
|
68
68
|
err_field(:schedule_type, options, "Not in #{DEFINED_JOB_TYPES}")
|
@@ -71,10 +71,10 @@ module Que
|
|
71
71
|
# rubocop:enable Style/GuardClause
|
72
72
|
|
73
73
|
def validate_fields_presence(options)
|
74
|
-
err_field(:name, options,
|
75
|
-
err_field(:job_class, options,
|
74
|
+
err_field(:name, options, "name must be present") if name.nil?
|
75
|
+
err_field(:job_class, options, "job_class must be present") if job_class.nil?
|
76
76
|
# An invalid cron is nil
|
77
|
-
err_field(:cron, options,
|
77
|
+
err_field(:cron, options, "cron must be present") if cron.nil?
|
78
78
|
end
|
79
79
|
|
80
80
|
def validate_job_class_related(options)
|
@@ -88,7 +88,7 @@ module Que
|
|
88
88
|
if queue &&
|
89
89
|
Que::Scheduler::ToEnqueue.active_job_sufficient_version? &&
|
90
90
|
job_class < ::ActiveJob::Base &&
|
91
|
-
Que::Scheduler::ToEnqueue.active_job_version < Gem::Version.create(
|
91
|
+
Que::Scheduler::ToEnqueue.active_job_version < Gem::Version.create("6.0.3")
|
92
92
|
puts <<-ERR
|
93
93
|
WARNING from que-scheduler....
|
94
94
|
Between versions 4.2.3 and 6.0.2 (inclusive) Rails did not support setting queue names
|
@@ -104,7 +104,7 @@ module Que
|
|
104
104
|
end
|
105
105
|
end
|
106
106
|
|
107
|
-
def err_field(field, options, reason =
|
107
|
+
def err_field(field, options, reason = "")
|
108
108
|
schedule = Que::Scheduler.configuration.schedule_location
|
109
109
|
value = options[field]
|
110
110
|
raise "Invalid #{field} '#{value}' for '#{name}' in que-scheduler schedule #{schedule}.\n" \
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "que"
|
2
2
|
|
3
3
|
# This job can optionally be scheduled to clear down the que-scheduler audit log if it
|
4
4
|
# isn't required in the long term.
|
@@ -21,8 +21,8 @@ module Que
|
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
|
-
DELETE_AUDIT_ENQUEUED_SQL = build_sql(
|
25
|
-
DELETE_AUDIT_SQL = build_sql(
|
24
|
+
DELETE_AUDIT_ENQUEUED_SQL = build_sql("que_scheduler_audit_enqueued").freeze
|
25
|
+
DELETE_AUDIT_SQL = build_sql("que_scheduler_audit").freeze
|
26
26
|
|
27
27
|
# Very low priority
|
28
28
|
Que::Scheduler::VersionSupport.set_priority(self, 100)
|
@@ -36,7 +36,7 @@ module Que
|
|
36
36
|
# This will delete all but `retain_row_count` oldest rows
|
37
37
|
count = Que::Scheduler::VersionSupport.execute(DELETE_AUDIT_SQL, [retain_row_count])
|
38
38
|
log = "#{self.class} cleared down #{count.first.fetch(:count)} rows"
|
39
|
-
::Que.log(event:
|
39
|
+
::Que.log(event: "que-scheduler".to_sym, message: log)
|
40
40
|
end
|
41
41
|
end
|
42
42
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
-- Ensure there is no more than one scheduler
|
2
|
+
CREATE UNIQUE INDEX que_scheduler_job_in_que_jobs_unique_index ON que_jobs(job_class)
|
3
|
+
WHERE job_class = 'Que::Scheduler::SchedulerJob';
|
4
|
+
|
5
|
+
-- Ensure there is at least one scheduler
|
6
|
+
CREATE OR REPLACE FUNCTION que_scheduler_check_job_exists() RETURNS bool AS $$
|
7
|
+
SELECT EXISTS(SELECT * FROM que_jobs WHERE job_class = 'Que::Scheduler::SchedulerJob');
|
8
|
+
$$ LANGUAGE SQL;
|
9
|
+
|
10
|
+
CREATE OR REPLACE FUNCTION que_scheduler_prevent_job_deletion() RETURNS TRIGGER AS
|
11
|
+
$BODY$
|
12
|
+
DECLARE
|
13
|
+
BEGIN
|
14
|
+
IF OLD.job_class = 'Que::Scheduler::SchedulerJob' THEN
|
15
|
+
IF NOT que_scheduler_check_job_exists() THEN
|
16
|
+
raise exception 'Deletion of que_scheduler job % prevented. Deleting the que_scheduler job is almost certainly a mistake.', OLD.job_id;
|
17
|
+
END IF;
|
18
|
+
END IF;
|
19
|
+
RETURN OLD;
|
20
|
+
END;
|
21
|
+
$BODY$
|
22
|
+
LANGUAGE 'plpgsql';
|
23
|
+
|
24
|
+
CREATE CONSTRAINT TRIGGER que_scheduler_prevent_job_deletion_trigger AFTER UPDATE OR DELETE ON que_jobs
|
25
|
+
DEFERRABLE INITIALLY DEFERRED
|
26
|
+
FOR EACH ROW EXECUTE PROCEDURE que_scheduler_prevent_job_deletion();
|
@@ -1,20 +1,37 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "configuration"
|
2
|
+
require_relative "defined_job"
|
2
3
|
|
3
4
|
module Que
|
4
5
|
module Scheduler
|
5
6
|
class Schedule
|
6
7
|
class << self
|
8
|
+
# The main method for determining the schedule. It has to evaluate the schedule as late as
|
9
|
+
# possible (ie just as it is about to be used) as we cannot guarantee we are in a Rails
|
10
|
+
# app with initializers. In a future release this may change to "fast fail" in Rails by
|
11
|
+
# checking the config up front.
|
7
12
|
def schedule
|
8
13
|
@schedule ||=
|
9
14
|
begin
|
10
|
-
|
11
|
-
|
15
|
+
configuration = Que::Scheduler.configuration
|
16
|
+
if !configuration.schedule.nil?
|
17
|
+
# If an explicit schedule as a hash has been defined, use that.
|
18
|
+
from_hash(configuration.schedule)
|
19
|
+
elsif File.exist?(configuration.schedule_location)
|
20
|
+
# If the schedule is defined as a file location, then load it and return it.
|
21
|
+
from_file(configuration.schedule_location)
|
22
|
+
else
|
23
|
+
raise "No que-scheduler config set, or file found " \
|
24
|
+
"at #{configuration.schedule_location}"
|
25
|
+
end
|
12
26
|
end
|
13
27
|
end
|
14
28
|
|
15
29
|
def from_file(location)
|
16
|
-
|
17
|
-
|
30
|
+
from_yaml(IO.read(location))
|
31
|
+
end
|
32
|
+
|
33
|
+
def from_yaml(config)
|
34
|
+
config_hash = YAML.safe_load(config)
|
18
35
|
from_hash(config_hash)
|
19
36
|
end
|
20
37
|
|
@@ -30,11 +47,11 @@ module Que
|
|
30
47
|
# an array with one item which is that value (this includes if it is a hash). It could
|
31
48
|
# also be a single nil value.
|
32
49
|
args_array =
|
33
|
-
if !defined_job_hash.key?(
|
50
|
+
if !defined_job_hash.key?("args")
|
34
51
|
# No args were requested
|
35
52
|
[]
|
36
53
|
else
|
37
|
-
args = defined_job_hash[
|
54
|
+
args = defined_job_hash["args"]
|
38
55
|
if args.is_a?(Array)
|
39
56
|
# An array of args was requested
|
40
57
|
args
|
@@ -47,12 +64,12 @@ module Que
|
|
47
64
|
|
48
65
|
Que::Scheduler::DefinedJob.create(
|
49
66
|
name: name,
|
50
|
-
job_class: defined_job_hash[
|
51
|
-
queue: defined_job_hash[
|
67
|
+
job_class: defined_job_hash["class"] || name,
|
68
|
+
queue: defined_job_hash["queue"],
|
52
69
|
args_array: args_array,
|
53
|
-
priority: defined_job_hash[
|
54
|
-
cron: defined_job_hash[
|
55
|
-
schedule_type: defined_job_hash[
|
70
|
+
priority: defined_job_hash["priority"],
|
71
|
+
cron: defined_job_hash["cron"],
|
72
|
+
schedule_type: defined_job_hash["schedule_type"]&.to_sym
|
56
73
|
)
|
57
74
|
end
|
58
75
|
end
|
@@ -1,11 +1,13 @@
|
|
1
|
-
require
|
1
|
+
require "que"
|
2
|
+
require "active_support"
|
3
|
+
require "active_support/core_ext/time"
|
2
4
|
|
3
|
-
require_relative
|
4
|
-
require_relative
|
5
|
-
require_relative
|
6
|
-
require_relative
|
7
|
-
require_relative
|
8
|
-
require_relative
|
5
|
+
require_relative "schedule"
|
6
|
+
require_relative "enqueueing_calculator"
|
7
|
+
require_relative "scheduler_job_args"
|
8
|
+
require_relative "state_checks"
|
9
|
+
require_relative "to_enqueue"
|
10
|
+
require_relative "version_support"
|
9
11
|
|
10
12
|
# The main job that runs every minute, determining what needs to be enqueued, enqueues the required
|
11
13
|
# jobs, then re-enqueues itself.
|
@@ -14,8 +16,8 @@ module Que
|
|
14
16
|
class SchedulerJob < Que::Job
|
15
17
|
SCHEDULER_FREQUENCY = 60
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
+
VersionSupport.set_priority(self, 0)
|
20
|
+
VersionSupport.apply_retry_semantics(self)
|
19
21
|
|
20
22
|
def run(options = nil)
|
21
23
|
Que::Scheduler::Db.transaction do
|
@@ -25,12 +27,13 @@ module Que
|
|
25
27
|
logs = ["que-scheduler last ran at #{scheduler_job_args.last_run_time}."]
|
26
28
|
result = EnqueueingCalculator.parse(Scheduler.schedule.values, scheduler_job_args)
|
27
29
|
enqueued_jobs = enqueue_required_jobs(result, logs)
|
30
|
+
# Remove this job and schedule self again
|
31
|
+
destroy
|
28
32
|
enqueue_self_again(
|
29
33
|
scheduler_job_args, scheduler_job_args.as_time, result.job_dictionary, enqueued_jobs
|
30
34
|
)
|
31
35
|
# Only now we're sure nothing errored, log the results
|
32
|
-
logs.each { |str| ::Que.log(event:
|
33
|
-
destroy
|
36
|
+
logs.each { |str| ::Que.log(event: "que-scheduler".to_sym, message: str) }
|
34
37
|
end
|
35
38
|
end
|
36
39
|
|
@@ -51,13 +54,13 @@ module Que
|
|
51
54
|
else
|
52
55
|
# This can happen if a middleware nixes the enqueue call
|
53
56
|
"que-scheduler called enqueue on #{to_enqueue.job_class} " \
|
54
|
-
|
57
|
+
"but it reported no job was scheduled. Has `enqueue` been overridden?"
|
55
58
|
end
|
56
59
|
end
|
57
60
|
|
58
61
|
def enqueue_self_again(scheduler_job_args, last_full_execution, job_dictionary, enqueued_jobs)
|
59
62
|
# Log last run...
|
60
|
-
job_id =
|
63
|
+
job_id = VersionSupport.job_attributes(self).fetch(:job_id)
|
61
64
|
Audit.append(job_id, scheduler_job_args.as_time, enqueued_jobs)
|
62
65
|
|
63
66
|
# And rerun...
|
@@ -70,8 +73,8 @@ module Que
|
|
70
73
|
)
|
71
74
|
|
72
75
|
# rubocop:disable Style/GuardClause This reads better as a conditional
|
73
|
-
unless
|
74
|
-
raise
|
76
|
+
unless enqueued_job && VersionSupport.job_attributes(enqueued_job).fetch(:job_id)
|
77
|
+
raise "SchedulerJob could not self-schedule. Has `.enqueue` been monkey patched?"
|
75
78
|
end
|
76
79
|
# rubocop:enable Style/GuardClause
|
77
80
|
end
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "hashie"
|
2
|
+
require "active_support"
|
3
|
+
require "active_support/time_with_zone"
|
4
4
|
|
5
5
|
# These are the args that are used for this particular run of the scheduler.
|
6
6
|
module Que
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require_relative
|
2
|
-
require_relative
|
3
|
-
require_relative
|
1
|
+
require_relative "audit"
|
2
|
+
require_relative "db"
|
3
|
+
require_relative "migrations"
|
4
4
|
|
5
5
|
module Que
|
6
6
|
module Scheduler
|
@@ -8,7 +8,6 @@ module Que
|
|
8
8
|
class << self
|
9
9
|
def check
|
10
10
|
assert_db_migrated
|
11
|
-
assert_one_scheduler_job
|
12
11
|
end
|
13
12
|
|
14
13
|
private
|
@@ -60,21 +59,6 @@ module Que
|
|
60
59
|
synchronously. This will fail as que-scheduler needs the above tables to work.
|
61
60
|
ERR
|
62
61
|
end
|
63
|
-
|
64
|
-
def assert_one_scheduler_job
|
65
|
-
schedulers = Que::Scheduler::Db.count_schedulers
|
66
|
-
return if schedulers == 1
|
67
|
-
|
68
|
-
raise(<<-ERR)
|
69
|
-
Only one #{Que::Scheduler::SchedulerJob.name} should be enqueued. #{schedulers} were found.
|
70
|
-
|
71
|
-
que-scheduler works by running a self-enqueueing version of itself that determines which
|
72
|
-
jobs should be enqueued based on the provided config. If two or more que-schedulers were
|
73
|
-
to run at once, then duplicate jobs would occur.
|
74
|
-
|
75
|
-
To resolve this problem, please remove any duplicate scheduler jobs from the que_jobs table.
|
76
|
-
ERR
|
77
|
-
end
|
78
62
|
end
|
79
63
|
end
|
80
64
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "que"
|
2
2
|
|
3
3
|
# This module uses polymorphic dispatch to centralise the differences between supporting Que::Job
|
4
4
|
# and other job systems.
|
@@ -23,13 +23,13 @@ module Que
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def active_job_version
|
26
|
-
Gem.loaded_specs[
|
26
|
+
Gem.loaded_specs["activejob"]&.version
|
27
27
|
end
|
28
28
|
|
29
29
|
def active_job_sufficient_version?
|
30
30
|
# ActiveJob 4.x does not support job_ids correctly
|
31
31
|
# https://github.com/rails/rails/pull/20056/files
|
32
|
-
active_job_version && active_job_version > Gem::Version.create(
|
32
|
+
active_job_version && active_job_version > Gem::Version.create("5")
|
33
33
|
end
|
34
34
|
|
35
35
|
def active_job_version_supports_queues?
|
@@ -38,7 +38,7 @@ module Que
|
|
38
38
|
# and readded in Rails 6.0.3
|
39
39
|
# https://github.com/rails/rails/pull/38635
|
40
40
|
ToEnqueue.active_job_version && ToEnqueue.active_job_version >=
|
41
|
-
Gem::Version.create(
|
41
|
+
Gem::Version.create("6.0.3")
|
42
42
|
end
|
43
43
|
|
44
44
|
private
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "que"
|
2
2
|
|
3
3
|
# The purpose of this module is to centralise the differences when supporting both que 0.x and
|
4
4
|
# 1.x with the same gem.
|
@@ -15,7 +15,7 @@ module Que
|
|
15
15
|
# the top of all jobs it enqueues.
|
16
16
|
def set_priority(context, priority)
|
17
17
|
if zero_major?
|
18
|
-
context.instance_variable_set(
|
18
|
+
context.instance_variable_set("@priority", priority)
|
19
19
|
else
|
20
20
|
context.priority = priority
|
21
21
|
end
|
@@ -24,7 +24,7 @@ module Que
|
|
24
24
|
# Ensure the job runs at least once an hour when it is backing off due to errors
|
25
25
|
def apply_retry_semantics(context)
|
26
26
|
if zero_major?
|
27
|
-
context.instance_variable_set(
|
27
|
+
context.instance_variable_set("@retry_interval", RETRY_PROC)
|
28
28
|
else
|
29
29
|
context.maximum_retry_count = 1 << 128 # Heat death of universe
|
30
30
|
context.retry_interval = RETRY_PROC
|
@@ -33,9 +33,9 @@ module Que
|
|
33
33
|
|
34
34
|
def job_attributes(enqueued_job)
|
35
35
|
if zero_major?
|
36
|
-
enqueued_job.attrs.transform_keys(&:to_sym)
|
36
|
+
enqueued_job.attrs.to_h.transform_keys(&:to_sym)
|
37
37
|
else
|
38
|
-
enqueued_job.que_attrs.transform_keys(&:to_sym).tap do |hash|
|
38
|
+
enqueued_job.que_attrs.to_h.transform_keys(&:to_sym).tap do |hash|
|
39
39
|
hash[:job_id] = hash.delete(:id)
|
40
40
|
end
|
41
41
|
end
|
@@ -48,7 +48,7 @@ module Que
|
|
48
48
|
end
|
49
49
|
|
50
50
|
def default_scheduler_queue
|
51
|
-
zero_major? ?
|
51
|
+
zero_major? ? "" : Que::DEFAULT_QUEUE
|
52
52
|
end
|
53
53
|
|
54
54
|
def running_synchronously?
|
@@ -56,18 +56,18 @@ module Que
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def running_synchronously_code?
|
59
|
-
zero_major? ?
|
59
|
+
zero_major? ? "Que.mode == :sync" : "Que.run_synchronously = true"
|
60
60
|
end
|
61
61
|
|
62
62
|
def zero_major?
|
63
63
|
# This is the only way to handle beta releases too
|
64
|
-
@zero_major ||= Gem.loaded_specs[
|
64
|
+
@zero_major ||= Gem.loaded_specs["que"].version.to_s.split(".").first.to_i.zero?
|
65
65
|
end
|
66
66
|
|
67
67
|
private
|
68
68
|
|
69
69
|
def normalise_array_of_hashes(array)
|
70
|
-
array.map { |row| row.transform_keys(&:to_sym) }
|
70
|
+
array.map { |row| row.to_h.transform_keys(&:to_sym) }
|
71
71
|
end
|
72
72
|
end
|
73
73
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que-scheduler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Harry Lascelles
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -112,6 +112,20 @@ dependencies:
|
|
112
112
|
- - ">="
|
113
113
|
- !ruby/object:Gem::Version
|
114
114
|
version: '0'
|
115
|
+
- !ruby/object:Gem::Dependency
|
116
|
+
name: climate_control
|
117
|
+
requirement: !ruby/object:Gem::Requirement
|
118
|
+
requirements:
|
119
|
+
- - ">="
|
120
|
+
- !ruby/object:Gem::Version
|
121
|
+
version: '0'
|
122
|
+
type: :development
|
123
|
+
prerelease: false
|
124
|
+
version_requirements: !ruby/object:Gem::Requirement
|
125
|
+
requirements:
|
126
|
+
- - ">="
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '0'
|
115
129
|
- !ruby/object:Gem::Dependency
|
116
130
|
name: combustion
|
117
131
|
requirement: !ruby/object:Gem::Requirement
|
@@ -319,7 +333,7 @@ files:
|
|
319
333
|
- lib/que-scheduler.rb
|
320
334
|
- lib/que/scheduler.rb
|
321
335
|
- lib/que/scheduler/audit.rb
|
322
|
-
- lib/que/scheduler/
|
336
|
+
- lib/que/scheduler/configuration.rb
|
323
337
|
- lib/que/scheduler/db.rb
|
324
338
|
- lib/que/scheduler/defined_job.rb
|
325
339
|
- lib/que/scheduler/enqueueing_calculator.rb
|
@@ -335,6 +349,8 @@ files:
|
|
335
349
|
- lib/que/scheduler/migrations/4/up.sql
|
336
350
|
- lib/que/scheduler/migrations/5/down.sql
|
337
351
|
- lib/que/scheduler/migrations/5/up.sql
|
352
|
+
- lib/que/scheduler/migrations/6/down.sql
|
353
|
+
- lib/que/scheduler/migrations/6/up.sql
|
338
354
|
- lib/que/scheduler/schedule.rb
|
339
355
|
- lib/que/scheduler/scheduler_job.rb
|
340
356
|
- lib/que/scheduler/scheduler_job_args.rb
|
data/lib/que/scheduler/config.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'que'
|
2
|
-
require_relative 'version_support'
|
3
|
-
|
4
|
-
module Que
|
5
|
-
module Scheduler
|
6
|
-
class << self
|
7
|
-
attr_accessor :configuration
|
8
|
-
end
|
9
|
-
|
10
|
-
def self.configure
|
11
|
-
self.configuration ||= Configuration.new
|
12
|
-
yield(configuration)
|
13
|
-
end
|
14
|
-
|
15
|
-
class Configuration
|
16
|
-
attr_accessor :schedule_location
|
17
|
-
attr_accessor :transaction_adapter
|
18
|
-
attr_accessor :que_scheduler_queue
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
Que::Scheduler.configure do |config|
|
24
|
-
config.schedule_location = ENV.fetch('QUE_SCHEDULER_CONFIG_LOCATION', 'config/que_schedule.yml')
|
25
|
-
config.transaction_adapter = ::Que.method(:transaction)
|
26
|
-
config.que_scheduler_queue = Que::Scheduler::VersionSupport.default_scheduler_queue
|
27
|
-
end
|