que-scheduler 3.3.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +84 -29
- data/lib/que-scheduler.rb +1 -1
- data/lib/que/scheduler.rb +8 -7
- data/lib/que/scheduler/audit.rb +3 -3
- data/lib/que/scheduler/config.rb +3 -3
- data/lib/que/scheduler/db.rb +2 -2
- data/lib/que/scheduler/defined_job.rb +10 -12
- data/lib/que/scheduler/enqueueing_calculator.rb +1 -1
- data/lib/que/scheduler/jobs/que_scheduler_audit_clear_down_job.rb +45 -0
- data/lib/que/scheduler/migrations/5/down.sql +1 -0
- data/lib/que/scheduler/migrations/5/up.sql +1 -0
- data/lib/que/scheduler/migrations/6/down.sql +7 -0
- data/lib/que/scheduler/migrations/6/up.sql +26 -0
- data/lib/que/scheduler/schedule.rb +40 -11
- data/lib/que/scheduler/scheduler_job.rb +20 -17
- data/lib/que/scheduler/scheduler_job_args.rb +3 -3
- data/lib/que/scheduler/state_checks.rb +3 -19
- data/lib/que/scheduler/to_enqueue.rb +15 -8
- data/lib/que/scheduler/version.rb +1 -1
- data/lib/que/scheduler/version_support.rb +9 -9
- metadata +26 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f6250ff590c3c280c0219d93ba57f85758a993fba1185d2064f1ba714c8c9e4c
|
4
|
+
data.tar.gz: cb49ce9c63314888e05f48cc7cd5a98817deb0df03763b9aae756055261acdbb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9516e812e2d394d5ff7c23641db0344a34c3560ad4ed02a114cc9820beee8202da9416450fbfc68f92bfaa6aed4f5bf16d1df3d2476f487a58f6f0fb32141aa1
|
7
|
+
data.tar.gz: 4d29b9a9161713935ccf1dcea71303195131fa8c27570b05a528fdec68d146b72c316ebeed523fabecda625c0128e4accca76ba0f47ba63fb8ff1d6106c96120
|
data/README.md
CHANGED
@@ -18,23 +18,28 @@ needs to be run, enqueueing those jobs, then enqueueing itself to check again la
|
|
18
18
|
```ruby
|
19
19
|
gem 'que-scheduler'
|
20
20
|
```
|
21
|
-
1. Specify a schedule in a yml file (see below). The default location that
|
22
|
-
look for it is `config/que_schedule.yml`.
|
23
|
-
files, but with additional features.
|
21
|
+
1. Specify a schedule in a yml file or programmatically (see below). The default location that
|
22
|
+
que-scheduler will look for it is `config/que_schedule.yml`. The format is essentially the same as
|
23
|
+
resque-scheduler files, but with additional features.
|
24
24
|
|
25
|
-
1. Add a migration to start the job scheduler and prepare the audit table. Note that this migration
|
25
|
+
1. Add a migration to start the job scheduler and prepare the audit table. Note that this migration
|
26
|
+
will fail if Que is set to execute jobs synchronously, i.e. `Que::Job.run_synchronously = true`.
|
26
27
|
|
27
28
|
```ruby
|
28
29
|
class CreateQueSchedulerSchema < ActiveRecord::Migration
|
29
30
|
def change
|
30
|
-
Que::Scheduler::Migrations.migrate!(version:
|
31
|
+
Que::Scheduler::Migrations.migrate!(version: 6)
|
31
32
|
end
|
32
33
|
end
|
33
34
|
```
|
34
35
|
|
35
36
|
## Schedule configuration
|
36
37
|
|
37
|
-
The schedule file
|
38
|
+
The schedule file should be placed here: `config/que_schedule.yml`. Alternatively if you
|
39
|
+
wish to generate the configuration dynamically, you can set it directly with
|
40
|
+
`Que::Scheduler.schedule = some_hash`.
|
41
|
+
|
42
|
+
The file is a list of que job classes with arguments and a schedule frequency (in crontab
|
38
43
|
syntax). The format is similar to the resque-scheduler format, though priorities must be supplied as
|
39
44
|
integers, and job classes must be migrated from Resque to Que. Cron syntax can be anything
|
40
45
|
understood by [fugit](https://github.com/floraison/fugit#fugitcron).
|
@@ -65,10 +70,21 @@ BatchOrders:
|
|
65
70
|
cron: "0 0 * * *"
|
66
71
|
priority: 25
|
67
72
|
|
68
|
-
# Specify job arguments
|
73
|
+
# Specify array job arguments
|
69
74
|
SendOrders:
|
70
75
|
cron: "0 0 * * *"
|
71
76
|
args: ['open']
|
77
|
+
|
78
|
+
# Specify hash job arguments. Note, this appears as a single hash to `run`, not as kwargs.
|
79
|
+
SendPreorders:
|
80
|
+
cron: "0 0 * * *"
|
81
|
+
args:
|
82
|
+
order_type: special
|
83
|
+
|
84
|
+
# Specify a single nil argument
|
85
|
+
SendPostorders:
|
86
|
+
cron: "0 0 * * *"
|
87
|
+
args: ~ # See https://stackoverflow.com/a/51990876/1267203
|
72
88
|
|
73
89
|
# Use simpler cron syntax
|
74
90
|
SendBilling:
|
@@ -136,8 +152,58 @@ end
|
|
136
152
|
|
137
153
|
## Scheduler Audit
|
138
154
|
|
139
|
-
An audit table
|
140
|
-
|
155
|
+
An audit table `que_scheduler_audit` is written to by the scheduler to keep a history of when the
|
156
|
+
scheduler ran to calculate what was necessary to run (if anything). It is created by the included
|
157
|
+
migration tasks.
|
158
|
+
|
159
|
+
Additionally, there is the audit table `que_scheduler_audit_enqueued`. This logs every job that
|
160
|
+
the scheduler enqueues.
|
161
|
+
|
162
|
+
When there is a major version (breaking) change, a migration should be run in. The version of the
|
163
|
+
latest migration proceeds at a faster rate than the version of the gem, eg if the gem is on version
|
164
|
+
3 then the migrations may be on version 6).
|
165
|
+
|
166
|
+
To run in all the migrations required up to a number, just migrate to that number with one line, and
|
167
|
+
it will perform all the intermediary steps.
|
168
|
+
|
169
|
+
ie, This will perform all migrations necessary up to the latest version, skipping any already
|
170
|
+
performed.
|
171
|
+
|
172
|
+
```ruby
|
173
|
+
class CreateQueSchedulerSchema < ActiveRecord::Migration
|
174
|
+
def change
|
175
|
+
Que::Scheduler::Migrations.migrate!(version: 6)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
```
|
179
|
+
|
180
|
+
The changes in past migrations were:
|
181
|
+
|
182
|
+
| Version | Changes |
|
183
|
+
|:-------:|---------------------------------------------------------------------------------|
|
184
|
+
| 1 | Enqueued the main Que::Scheduler. This is the job that performs the scheduling. |
|
185
|
+
| 2 | Added the audit table `que_scheduler_audit`. |
|
186
|
+
| 3 | Added the audit table `que_scheduler_audit_enqueued`. |
|
187
|
+
| 4 | Updated the the audit tables to use bigints |
|
188
|
+
| 5 | Dropped an unnecessary index |
|
189
|
+
| 6 | Enforced single scheduler job at the trigger level |
|
190
|
+
|
191
|
+
## Built in optional job for audit clear down
|
192
|
+
|
193
|
+
que-scheduler comes with the `QueSchedulerAuditClearDownJob` job built in that you can optionally
|
194
|
+
schedule to clear down audit rows if you don't need to retain them indefinitely. You should add this
|
195
|
+
to your own scheduler config yaml.
|
196
|
+
|
197
|
+
For example:
|
198
|
+
|
199
|
+
```yaml
|
200
|
+
# This will clear down the oldest que-scheduler audit rows. Since que-scheduler
|
201
|
+
# runs approximately every minute, 129600 is 90 days.
|
202
|
+
Que::Scheduler::Jobs::QueSchedulerAuditClearDownJob:
|
203
|
+
cron: "0 0 * * *"
|
204
|
+
args:
|
205
|
+
retain_row_count: 129600
|
206
|
+
```
|
141
207
|
|
142
208
|
## HA Redundancy and DB restores
|
143
209
|
|
@@ -152,8 +218,8 @@ in a coherent state with the rest of your database.
|
|
152
218
|
## Concurrent scheduler detection
|
153
219
|
|
154
220
|
No matter how many tasks you have defined in your schedule, you will only ever need one que-scheduler
|
155
|
-
job enqueued. que-scheduler knows this, and
|
156
|
-
|
221
|
+
job enqueued. que-scheduler knows this, and there are DB constraints in place to ensure there is
|
222
|
+
only ever exactly one scheduler job.
|
157
223
|
|
158
224
|
It also follows que job design [best practices](https://github.com/chanks/que/blob/master/docs/writing_reliable_jobs.md),
|
159
225
|
using ACID guarantees, to ensure that it will never run multiple times. If the scheduler crashes for any reason,
|
@@ -172,23 +238,6 @@ then reschedules itself. The flow is as follows:
|
|
172
238
|
1. After a deploy that changes the schedule, the job notices any new jobs to schedule, and knows which
|
173
239
|
ones to forget. It does not need to be re-enqueued or restarted.
|
174
240
|
|
175
|
-
## DB Migrations
|
176
|
-
|
177
|
-
When there is a major version (breaking) change, a migration should be run in. The version of the
|
178
|
-
migration proceeds at a faster rate than the version of the gem. To run in all the migrations required
|
179
|
-
up to a number, just migrate to that number with one line, and it will perform all the intermediary steps.
|
180
|
-
|
181
|
-
ie, `Que::Scheduler::Migrations.migrate!(version: 4)` will perform all migrations necessary to
|
182
|
-
reach migration version `4`.
|
183
|
-
|
184
|
-
As of migration `4`, two elements are added to the DB for que-scheduler to run.
|
185
|
-
|
186
|
-
1. The first is the scheduler job itself, which runs forever, re-enqueuing itself to performs its
|
187
|
-
duties.
|
188
|
-
1. The second part comprises the audit table `que_scheduler_audit` and the "enqueued" table
|
189
|
-
`que_scheduler_audit_enqueued`. The first tracks when the scheduler calculated what was necessary to run
|
190
|
-
(if anything). The second then logs every job that the scheduler enqueues.
|
191
|
-
|
192
241
|
## Testing Configuration
|
193
242
|
|
194
243
|
You can add tests to validate your configuration during the spec phase. This will perform a variety
|
@@ -222,6 +271,10 @@ The scheduler will then continue to retry indefinitely.
|
|
222
271
|
que-scheduler uses [semantic versioning](https://semver.org/), so major version changes will usually
|
223
272
|
require additional actions to be taken upgrading from one major version to another.
|
224
273
|
|
274
|
+
## Changelog
|
275
|
+
|
276
|
+
A full changelog can be found here: [CHANGELOG.md](https://github.com/hlascelles/que-scheduler/blob/master/CHANGELOG.md)
|
277
|
+
|
225
278
|
## System requirements
|
226
279
|
|
227
280
|
Your [postgres](https://www.postgresql.org/) database must be at least version 9.4.0.
|
@@ -232,7 +285,9 @@ This gem was inspired by the makers of the excellent [Que](https://github.com/ch
|
|
232
285
|
|
233
286
|
## Contributors
|
234
287
|
|
288
|
+
* @bnauta
|
289
|
+
* @JackDanger
|
235
290
|
* @jish
|
236
291
|
* @joehorsnell
|
237
|
-
* @
|
292
|
+
* @krzyzak
|
238
293
|
* @papodaca
|
data/lib/que-scheduler.rb
CHANGED
data/lib/que/scheduler.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
1
|
+
require "que/scheduler/version"
|
2
|
+
require "que/scheduler/version_support"
|
3
|
+
require "que/scheduler/config"
|
4
|
+
require "que/scheduler/scheduler_job"
|
5
|
+
require "que/scheduler/db"
|
6
|
+
require "que/scheduler/audit"
|
7
|
+
require "que/scheduler/migrations"
|
8
|
+
require "que/scheduler/jobs/que_scheduler_audit_clear_down_job"
|
data/lib/que/scheduler/audit.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative "to_enqueue"
|
4
4
|
|
5
5
|
module Que
|
6
6
|
module Scheduler
|
7
7
|
module Audit
|
8
|
-
TABLE_NAME =
|
9
|
-
ENQUEUED_TABLE_NAME =
|
8
|
+
TABLE_NAME = "que_scheduler_audit"
|
9
|
+
ENQUEUED_TABLE_NAME = "que_scheduler_audit_enqueued"
|
10
10
|
INSERT_AUDIT = %{
|
11
11
|
INSERT INTO #{TABLE_NAME} (scheduler_job_id, executed_at)
|
12
12
|
VALUES ($1::bigint, $2::timestamptz)
|
data/lib/que/scheduler/config.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require_relative
|
1
|
+
require "que"
|
2
|
+
require_relative "version_support"
|
3
3
|
|
4
4
|
module Que
|
5
5
|
module Scheduler
|
@@ -21,7 +21,7 @@ module Que
|
|
21
21
|
end
|
22
22
|
|
23
23
|
Que::Scheduler.configure do |config|
|
24
|
-
config.schedule_location = ENV.fetch(
|
24
|
+
config.schedule_location = ENV.fetch("QUE_SCHEDULER_CONFIG_LOCATION", "config/que_schedule.yml")
|
25
25
|
config.transaction_adapter = ::Que.method(:transaction)
|
26
26
|
config.que_scheduler_queue = Que::Scheduler::VersionSupport.default_scheduler_queue
|
27
27
|
end
|
data/lib/que/scheduler/db.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative "config"
|
4
4
|
|
5
5
|
module Que
|
6
6
|
module Scheduler
|
7
7
|
module Db
|
8
8
|
SCHEDULER_COUNT_SQL =
|
9
9
|
"SELECT COUNT(*) FROM que_jobs WHERE job_class = 'Que::Scheduler::SchedulerJob'"
|
10
|
-
NOW_SQL =
|
10
|
+
NOW_SQL = "SELECT now()"
|
11
11
|
|
12
12
|
class << self
|
13
13
|
def count_schedulers
|
@@ -1,6 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require 'backports/2.4.0/hash/compact'
|
1
|
+
require "hashie"
|
2
|
+
require "fugit"
|
4
3
|
|
5
4
|
# This is the definition of one scheduleable job in the que-scheduler config yml file.
|
6
5
|
module Que
|
@@ -18,7 +17,7 @@ module Que
|
|
18
17
|
property :cron, transform_with: ->(v) { Fugit::Cron.parse(v) }
|
19
18
|
property :queue
|
20
19
|
property :priority
|
21
|
-
property :
|
20
|
+
property :args_array
|
22
21
|
property :schedule_type, default: DEFINED_JOB_TYPE_DEFAULT
|
23
22
|
|
24
23
|
class << self
|
@@ -60,10 +59,10 @@ module Que
|
|
60
59
|
# rubocop:disable Style/GuardClause This reads better as a conditional
|
61
60
|
def validate_fields_types(options)
|
62
61
|
unless queue.nil? || queue.is_a?(String)
|
63
|
-
err_field(:queue, options,
|
62
|
+
err_field(:queue, options, "queue must be a string")
|
64
63
|
end
|
65
64
|
unless priority.nil? || priority.is_a?(Integer)
|
66
|
-
err_field(:priority, options,
|
65
|
+
err_field(:priority, options, "priority must be an integer")
|
67
66
|
end
|
68
67
|
unless DEFINED_JOB_TYPES.include?(schedule_type)
|
69
68
|
err_field(:schedule_type, options, "Not in #{DEFINED_JOB_TYPES}")
|
@@ -72,10 +71,10 @@ module Que
|
|
72
71
|
# rubocop:enable Style/GuardClause
|
73
72
|
|
74
73
|
def validate_fields_presence(options)
|
75
|
-
err_field(:name, options,
|
76
|
-
err_field(:job_class, options,
|
74
|
+
err_field(:name, options, "name must be present") if name.nil?
|
75
|
+
err_field(:job_class, options, "job_class must be present") if job_class.nil?
|
77
76
|
# An invalid cron is nil
|
78
|
-
err_field(:cron, options,
|
77
|
+
err_field(:cron, options, "cron must be present") if cron.nil?
|
79
78
|
end
|
80
79
|
|
81
80
|
def validate_job_class_related(options)
|
@@ -89,7 +88,7 @@ module Que
|
|
89
88
|
if queue &&
|
90
89
|
Que::Scheduler::ToEnqueue.active_job_sufficient_version? &&
|
91
90
|
job_class < ::ActiveJob::Base &&
|
92
|
-
Que::Scheduler::ToEnqueue.active_job_version < Gem::Version.create(
|
91
|
+
Que::Scheduler::ToEnqueue.active_job_version < Gem::Version.create("6.0.3")
|
93
92
|
puts <<-ERR
|
94
93
|
WARNING from que-scheduler....
|
95
94
|
Between versions 4.2.3 and 6.0.2 (inclusive) Rails did not support setting queue names
|
@@ -105,7 +104,7 @@ module Que
|
|
105
104
|
end
|
106
105
|
end
|
107
106
|
|
108
|
-
def err_field(field, options, reason =
|
107
|
+
def err_field(field, options, reason = "")
|
109
108
|
schedule = Que::Scheduler.configuration.schedule_location
|
110
109
|
value = options[field]
|
111
110
|
raise "Invalid #{field} '#{value}' for '#{name}' in que-scheduler schedule #{schedule}.\n" \
|
@@ -116,7 +115,6 @@ module Que
|
|
116
115
|
return [] if missed_times.empty?
|
117
116
|
|
118
117
|
options = to_h.slice(:args, :queue, :priority, :job_class).compact
|
119
|
-
args_array = args.is_a?(Array) ? args : Array(args)
|
120
118
|
|
121
119
|
if schedule_type == DefinedJob::DEFINED_JOB_TYPE_EVERY_EVENT
|
122
120
|
missed_times.map do |time_missed|
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require "que"
|
2
|
+
|
3
|
+
# This job can optionally be scheduled to clear down the que-scheduler audit log if it
|
4
|
+
# isn't required in the long term.
|
5
|
+
module Que
|
6
|
+
module Scheduler
|
7
|
+
module Jobs
|
8
|
+
class QueSchedulerAuditClearDownJob < Que::Job
|
9
|
+
class << self
|
10
|
+
def build_sql(table_name)
|
11
|
+
<<-SQL
|
12
|
+
WITH deleted AS (
|
13
|
+
DELETE FROM #{table_name}
|
14
|
+
WHERE scheduler_job_id <= (
|
15
|
+
SELECT scheduler_job_id FROM que_scheduler_audit
|
16
|
+
ORDER BY scheduler_job_id DESC
|
17
|
+
LIMIT 1 OFFSET $1
|
18
|
+
) RETURNING *
|
19
|
+
) SELECT count(*) FROM deleted;
|
20
|
+
SQL
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
DELETE_AUDIT_ENQUEUED_SQL = build_sql("que_scheduler_audit_enqueued").freeze
|
25
|
+
DELETE_AUDIT_SQL = build_sql("que_scheduler_audit").freeze
|
26
|
+
|
27
|
+
# Very low priority
|
28
|
+
Que::Scheduler::VersionSupport.set_priority(self, 100)
|
29
|
+
|
30
|
+
def run(options)
|
31
|
+
retain_row_count = options.fetch(:retain_row_count)
|
32
|
+
Que::Scheduler::Db.transaction do
|
33
|
+
# This may delete zero or more than `retain_row_count` depending on if anything was
|
34
|
+
# scheduled in each of the past schedule runs
|
35
|
+
Que::Scheduler::VersionSupport.execute(DELETE_AUDIT_ENQUEUED_SQL, [retain_row_count])
|
36
|
+
# This will delete all but `retain_row_count` oldest rows
|
37
|
+
count = Que::Scheduler::VersionSupport.execute(DELETE_AUDIT_SQL, [retain_row_count])
|
38
|
+
log = "#{self.class} cleared down #{count.first.fetch(:count)} rows"
|
39
|
+
::Que.log(event: "que-scheduler".to_sym, message: log)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
CREATE UNIQUE INDEX index_que_scheduler_audit_on_scheduler_job_id ON que_scheduler_audit USING btree (scheduler_job_id);
|
@@ -0,0 +1 @@
|
|
1
|
+
DROP INDEX index_que_scheduler_audit_on_scheduler_job_id;
|
@@ -0,0 +1,26 @@
|
|
1
|
+
-- Ensure there is no more than one scheduler
|
2
|
+
CREATE UNIQUE INDEX que_scheduler_job_in_que_jobs_unique_index ON que_jobs(job_class)
|
3
|
+
WHERE job_class = 'Que::Scheduler::SchedulerJob';
|
4
|
+
|
5
|
+
-- Ensure there is at least one scheduler
|
6
|
+
CREATE OR REPLACE FUNCTION que_scheduler_check_job_exists() RETURNS bool AS $$
|
7
|
+
SELECT EXISTS(SELECT * FROM que_jobs WHERE job_class = 'Que::Scheduler::SchedulerJob');
|
8
|
+
$$ LANGUAGE SQL;
|
9
|
+
|
10
|
+
CREATE OR REPLACE FUNCTION que_scheduler_prevent_job_deletion() RETURNS TRIGGER AS
|
11
|
+
$BODY$
|
12
|
+
DECLARE
|
13
|
+
BEGIN
|
14
|
+
IF OLD.job_class = 'Que::Scheduler::SchedulerJob' THEN
|
15
|
+
IF NOT que_scheduler_check_job_exists() THEN
|
16
|
+
raise exception 'Deletion of que_scheduler job % prevented. Deleting the que_scheduler job is almost certainly a mistake.', OLD.job_id;
|
17
|
+
END IF;
|
18
|
+
END IF;
|
19
|
+
RETURN OLD;
|
20
|
+
END;
|
21
|
+
$BODY$
|
22
|
+
LANGUAGE 'plpgsql';
|
23
|
+
|
24
|
+
CREATE CONSTRAINT TRIGGER que_scheduler_prevent_job_deletion_trigger AFTER UPDATE OR DELETE ON que_jobs
|
25
|
+
DEFERRABLE INITIALLY DEFERRED
|
26
|
+
FOR EACH ROW EXECUTE PROCEDURE que_scheduler_prevent_job_deletion();
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "defined_job"
|
2
2
|
|
3
3
|
module Que
|
4
4
|
module Scheduler
|
@@ -12,9 +12,16 @@ module Que
|
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
+
def schedule=(schedule_config)
|
16
|
+
@schedule = schedule_config.nil? ? nil : from_yaml(schedule_config)
|
17
|
+
end
|
18
|
+
|
15
19
|
def from_file(location)
|
16
|
-
|
17
|
-
|
20
|
+
from_yaml(IO.read(location))
|
21
|
+
end
|
22
|
+
|
23
|
+
def from_yaml(config)
|
24
|
+
config_hash = YAML.safe_load(config)
|
18
25
|
from_hash(config_hash)
|
19
26
|
end
|
20
27
|
|
@@ -24,17 +31,35 @@ module Que
|
|
24
31
|
end.to_h
|
25
32
|
end
|
26
33
|
|
27
|
-
private
|
28
|
-
|
29
34
|
def hash_item_to_defined_job(name, defined_job_hash)
|
35
|
+
# Que stores arguments as a json array. If the args we have to provide are already an
|
36
|
+
# array we can can simply pass them through. If it is a single non-nil value, then we make
|
37
|
+
# an array with one item which is that value (this includes if it is a hash). It could
|
38
|
+
# also be a single nil value.
|
39
|
+
args_array =
|
40
|
+
if !defined_job_hash.key?("args")
|
41
|
+
# No args were requested
|
42
|
+
[]
|
43
|
+
else
|
44
|
+
args = defined_job_hash["args"]
|
45
|
+
if args.is_a?(Array)
|
46
|
+
# An array of args was requested
|
47
|
+
args
|
48
|
+
else
|
49
|
+
# A single value, a nil, or a hash was requested. que expects this to
|
50
|
+
# be enqueued as an array of 1 item
|
51
|
+
[args]
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
30
55
|
Que::Scheduler::DefinedJob.create(
|
31
56
|
name: name,
|
32
|
-
job_class: defined_job_hash[
|
33
|
-
queue: defined_job_hash[
|
34
|
-
|
35
|
-
priority: defined_job_hash[
|
36
|
-
cron: defined_job_hash[
|
37
|
-
schedule_type: defined_job_hash[
|
57
|
+
job_class: defined_job_hash["class"] || name,
|
58
|
+
queue: defined_job_hash["queue"],
|
59
|
+
args_array: args_array,
|
60
|
+
priority: defined_job_hash["priority"],
|
61
|
+
cron: defined_job_hash["cron"],
|
62
|
+
schedule_type: defined_job_hash["schedule_type"]&.to_sym
|
38
63
|
)
|
39
64
|
end
|
40
65
|
end
|
@@ -44,6 +69,10 @@ module Que
|
|
44
69
|
def schedule
|
45
70
|
Schedule.schedule
|
46
71
|
end
|
72
|
+
|
73
|
+
def schedule=(value)
|
74
|
+
Schedule.schedule = value
|
75
|
+
end
|
47
76
|
end
|
48
77
|
end
|
49
78
|
end
|
@@ -1,11 +1,13 @@
|
|
1
|
-
require
|
1
|
+
require "que"
|
2
|
+
require "active_support"
|
3
|
+
require "active_support/core_ext/time"
|
2
4
|
|
3
|
-
require_relative
|
4
|
-
require_relative
|
5
|
-
require_relative
|
6
|
-
require_relative
|
7
|
-
require_relative
|
8
|
-
require_relative
|
5
|
+
require_relative "schedule"
|
6
|
+
require_relative "enqueueing_calculator"
|
7
|
+
require_relative "scheduler_job_args"
|
8
|
+
require_relative "state_checks"
|
9
|
+
require_relative "to_enqueue"
|
10
|
+
require_relative "version_support"
|
9
11
|
|
10
12
|
# The main job that runs every minute, determining what needs to be enqueued, enqueues the required
|
11
13
|
# jobs, then re-enqueues itself.
|
@@ -14,8 +16,8 @@ module Que
|
|
14
16
|
class SchedulerJob < Que::Job
|
15
17
|
SCHEDULER_FREQUENCY = 60
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
+
VersionSupport.set_priority(self, 0)
|
20
|
+
VersionSupport.apply_retry_semantics(self)
|
19
21
|
|
20
22
|
def run(options = nil)
|
21
23
|
Que::Scheduler::Db.transaction do
|
@@ -25,17 +27,18 @@ module Que
|
|
25
27
|
logs = ["que-scheduler last ran at #{scheduler_job_args.last_run_time}."]
|
26
28
|
result = EnqueueingCalculator.parse(Scheduler.schedule.values, scheduler_job_args)
|
27
29
|
enqueued_jobs = enqueue_required_jobs(result, logs)
|
30
|
+
# Remove this job and schedule self again
|
31
|
+
destroy
|
28
32
|
enqueue_self_again(
|
29
33
|
scheduler_job_args, scheduler_job_args.as_time, result.job_dictionary, enqueued_jobs
|
30
34
|
)
|
31
35
|
# Only now we're sure nothing errored, log the results
|
32
|
-
logs.each { |str| ::Que.log(event:
|
33
|
-
destroy
|
36
|
+
logs.each { |str| ::Que.log(event: "que-scheduler".to_sym, message: str) }
|
34
37
|
end
|
35
38
|
end
|
36
39
|
|
37
|
-
def enqueue_required_jobs(
|
38
|
-
|
40
|
+
def enqueue_required_jobs(calculator_result, logs)
|
41
|
+
calculator_result.missed_jobs.map do |to_enqueue|
|
39
42
|
to_enqueue.enqueue.tap do |enqueued_job|
|
40
43
|
check_enqueued_job(to_enqueue, enqueued_job, logs)
|
41
44
|
end
|
@@ -51,13 +54,13 @@ module Que
|
|
51
54
|
else
|
52
55
|
# This can happen if a middleware nixes the enqueue call
|
53
56
|
"que-scheduler called enqueue on #{to_enqueue.job_class} " \
|
54
|
-
|
57
|
+
"but it reported no job was scheduled. Has `enqueue` been overridden?"
|
55
58
|
end
|
56
59
|
end
|
57
60
|
|
58
61
|
def enqueue_self_again(scheduler_job_args, last_full_execution, job_dictionary, enqueued_jobs)
|
59
62
|
# Log last run...
|
60
|
-
job_id =
|
63
|
+
job_id = VersionSupport.job_attributes(self).fetch(:job_id)
|
61
64
|
Audit.append(job_id, scheduler_job_args.as_time, enqueued_jobs)
|
62
65
|
|
63
66
|
# And rerun...
|
@@ -70,8 +73,8 @@ module Que
|
|
70
73
|
)
|
71
74
|
|
72
75
|
# rubocop:disable Style/GuardClause This reads better as a conditional
|
73
|
-
unless
|
74
|
-
raise
|
76
|
+
unless enqueued_job && VersionSupport.job_attributes(enqueued_job).fetch(:job_id)
|
77
|
+
raise "SchedulerJob could not self-schedule. Has `.enqueue` been monkey patched?"
|
75
78
|
end
|
76
79
|
# rubocop:enable Style/GuardClause
|
77
80
|
end
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "hashie"
|
2
|
+
require "active_support"
|
3
|
+
require "active_support/time_with_zone"
|
4
4
|
|
5
5
|
# These are the args that are used for this particular run of the scheduler.
|
6
6
|
module Que
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require_relative
|
2
|
-
require_relative
|
3
|
-
require_relative
|
1
|
+
require_relative "audit"
|
2
|
+
require_relative "db"
|
3
|
+
require_relative "migrations"
|
4
4
|
|
5
5
|
module Que
|
6
6
|
module Scheduler
|
@@ -8,7 +8,6 @@ module Que
|
|
8
8
|
class << self
|
9
9
|
def check
|
10
10
|
assert_db_migrated
|
11
|
-
assert_one_scheduler_job
|
12
11
|
end
|
13
12
|
|
14
13
|
private
|
@@ -60,21 +59,6 @@ module Que
|
|
60
59
|
synchronously. This will fail as que-scheduler needs the above tables to work.
|
61
60
|
ERR
|
62
61
|
end
|
63
|
-
|
64
|
-
def assert_one_scheduler_job
|
65
|
-
schedulers = Que::Scheduler::Db.count_schedulers
|
66
|
-
return if schedulers == 1
|
67
|
-
|
68
|
-
raise(<<-ERR)
|
69
|
-
Only one #{Que::Scheduler::SchedulerJob.name} should be enqueued. #{schedulers} were found.
|
70
|
-
|
71
|
-
que-scheduler works by running a self-enqueueing version of itself that determines which
|
72
|
-
jobs should be enqueued based on the provided config. If two or more que-schedulers were
|
73
|
-
to run at once, then duplicate jobs would occur.
|
74
|
-
|
75
|
-
To resolve this problem, please remove any duplicate scheduler jobs from the que_jobs table.
|
76
|
-
ERR
|
77
|
-
end
|
78
62
|
end
|
79
63
|
end
|
80
64
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "que"
|
2
2
|
|
3
3
|
# This module uses polymorphic dispatch to centralise the differences between supporting Que::Job
|
4
4
|
# and other job systems.
|
@@ -23,13 +23,22 @@ module Que
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def active_job_version
|
26
|
-
Gem.loaded_specs[
|
26
|
+
Gem.loaded_specs["activejob"]&.version
|
27
27
|
end
|
28
28
|
|
29
29
|
def active_job_sufficient_version?
|
30
30
|
# ActiveJob 4.x does not support job_ids correctly
|
31
31
|
# https://github.com/rails/rails/pull/20056/files
|
32
|
-
active_job_version && active_job_version > Gem::Version.create(
|
32
|
+
active_job_version && active_job_version > Gem::Version.create("5")
|
33
|
+
end
|
34
|
+
|
35
|
+
def active_job_version_supports_queues?
|
36
|
+
# Supporting queue name in ActiveJob was removed in Rails 4.2.3
|
37
|
+
# https://github.com/rails/rails/pull/19498
|
38
|
+
# and readded in Rails 6.0.3
|
39
|
+
# https://github.com/rails/rails/pull/38635
|
40
|
+
ToEnqueue.active_job_version && ToEnqueue.active_job_version >=
|
41
|
+
Gem::Version.create("6.0.3")
|
33
42
|
end
|
34
43
|
|
35
44
|
private
|
@@ -87,8 +96,6 @@ module Que
|
|
87
96
|
EnqueuedJobType.new(enqueued_values)
|
88
97
|
end
|
89
98
|
|
90
|
-
private
|
91
|
-
|
92
99
|
def calculate_enqueued_values(job)
|
93
100
|
# Now read the just inserted job back out of the DB to get the actual values that will
|
94
101
|
# be used when the job is worked.
|
@@ -101,9 +108,8 @@ module Que
|
|
101
108
|
scheduled_at_float ? Time.zone.at(scheduled_at_float) : nil
|
102
109
|
end
|
103
110
|
|
104
|
-
# Rails
|
105
|
-
|
106
|
-
used_queue = nil
|
111
|
+
# Rails didn't support queues for ActiveJob for a while
|
112
|
+
used_queue = data[:queue_name] if ToEnqueue.active_job_version_supports_queues?
|
107
113
|
|
108
114
|
# We can't get the priority out of the DB, as the returned `job` doesn't give us access
|
109
115
|
# to the underlying ActiveJob that was scheduled. We have no option but to assume
|
@@ -125,6 +131,7 @@ module Que
|
|
125
131
|
job_settings = {
|
126
132
|
priority: priority,
|
127
133
|
wait_until: run_at,
|
134
|
+
queue: queue || Que::Scheduler::VersionSupport.default_scheduler_queue,
|
128
135
|
}.compact
|
129
136
|
|
130
137
|
job_class_set = job_class.set(**job_settings)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "que"
|
2
2
|
|
3
3
|
# The purpose of this module is to centralise the differences when supporting both que 0.x and
|
4
4
|
# 1.x with the same gem.
|
@@ -15,7 +15,7 @@ module Que
|
|
15
15
|
# the top of all jobs it enqueues.
|
16
16
|
def set_priority(context, priority)
|
17
17
|
if zero_major?
|
18
|
-
context.instance_variable_set(
|
18
|
+
context.instance_variable_set("@priority", priority)
|
19
19
|
else
|
20
20
|
context.priority = priority
|
21
21
|
end
|
@@ -24,7 +24,7 @@ module Que
|
|
24
24
|
# Ensure the job runs at least once an hour when it is backing off due to errors
|
25
25
|
def apply_retry_semantics(context)
|
26
26
|
if zero_major?
|
27
|
-
context.instance_variable_set(
|
27
|
+
context.instance_variable_set("@retry_interval", RETRY_PROC)
|
28
28
|
else
|
29
29
|
context.maximum_retry_count = 1 << 128 # Heat death of universe
|
30
30
|
context.retry_interval = RETRY_PROC
|
@@ -33,9 +33,9 @@ module Que
|
|
33
33
|
|
34
34
|
def job_attributes(enqueued_job)
|
35
35
|
if zero_major?
|
36
|
-
enqueued_job.attrs.transform_keys(&:to_sym)
|
36
|
+
enqueued_job.attrs.to_h.transform_keys(&:to_sym)
|
37
37
|
else
|
38
|
-
enqueued_job.que_attrs.transform_keys(&:to_sym).tap do |hash|
|
38
|
+
enqueued_job.que_attrs.to_h.transform_keys(&:to_sym).tap do |hash|
|
39
39
|
hash[:job_id] = hash.delete(:id)
|
40
40
|
end
|
41
41
|
end
|
@@ -48,7 +48,7 @@ module Que
|
|
48
48
|
end
|
49
49
|
|
50
50
|
def default_scheduler_queue
|
51
|
-
zero_major? ?
|
51
|
+
zero_major? ? "" : Que::DEFAULT_QUEUE
|
52
52
|
end
|
53
53
|
|
54
54
|
def running_synchronously?
|
@@ -56,18 +56,18 @@ module Que
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def running_synchronously_code?
|
59
|
-
zero_major? ?
|
59
|
+
zero_major? ? "Que.mode == :sync" : "Que.run_synchronously = true"
|
60
60
|
end
|
61
61
|
|
62
62
|
def zero_major?
|
63
63
|
# This is the only way to handle beta releases too
|
64
|
-
@zero_major ||= Gem.loaded_specs[
|
64
|
+
@zero_major ||= Gem.loaded_specs["que"].version.to_s.split(".").first.to_i.zero?
|
65
65
|
end
|
66
66
|
|
67
67
|
private
|
68
68
|
|
69
69
|
def normalise_array_of_hashes(array)
|
70
|
-
array.map { |row| row.transform_keys(&:to_sym) }
|
70
|
+
array.map { |row| row.to_h.transform_keys(&:to_sym) }
|
71
71
|
end
|
72
72
|
end
|
73
73
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que-scheduler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Harry Lascelles
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -24,20 +24,6 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4.0'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: backports
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - "~>"
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '3.10'
|
34
|
-
type: :runtime
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - "~>"
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '3.10'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: fugit
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -258,14 +244,28 @@ dependencies:
|
|
258
244
|
requirements:
|
259
245
|
- - '='
|
260
246
|
- !ruby/object:Gem::Version
|
261
|
-
version: 0.
|
247
|
+
version: 0.84.0
|
262
248
|
type: :development
|
263
249
|
prerelease: false
|
264
250
|
version_requirements: !ruby/object:Gem::Requirement
|
265
251
|
requirements:
|
266
252
|
- - '='
|
267
253
|
- !ruby/object:Gem::Version
|
268
|
-
version: 0.
|
254
|
+
version: 0.84.0
|
255
|
+
- !ruby/object:Gem::Dependency
|
256
|
+
name: rubocop-rspec
|
257
|
+
requirement: !ruby/object:Gem::Requirement
|
258
|
+
requirements:
|
259
|
+
- - ">="
|
260
|
+
- !ruby/object:Gem::Version
|
261
|
+
version: '0'
|
262
|
+
type: :development
|
263
|
+
prerelease: false
|
264
|
+
version_requirements: !ruby/object:Gem::Requirement
|
265
|
+
requirements:
|
266
|
+
- - ">="
|
267
|
+
- !ruby/object:Gem::Version
|
268
|
+
version: '0'
|
269
269
|
- !ruby/object:Gem::Dependency
|
270
270
|
name: sqlite3
|
271
271
|
requirement: !ruby/object:Gem::Requirement
|
@@ -323,6 +323,7 @@ files:
|
|
323
323
|
- lib/que/scheduler/db.rb
|
324
324
|
- lib/que/scheduler/defined_job.rb
|
325
325
|
- lib/que/scheduler/enqueueing_calculator.rb
|
326
|
+
- lib/que/scheduler/jobs/que_scheduler_audit_clear_down_job.rb
|
326
327
|
- lib/que/scheduler/migrations.rb
|
327
328
|
- lib/que/scheduler/migrations/1/down.sql
|
328
329
|
- lib/que/scheduler/migrations/1/up.sql
|
@@ -332,6 +333,10 @@ files:
|
|
332
333
|
- lib/que/scheduler/migrations/3/up.sql
|
333
334
|
- lib/que/scheduler/migrations/4/down.sql
|
334
335
|
- lib/que/scheduler/migrations/4/up.sql
|
336
|
+
- lib/que/scheduler/migrations/5/down.sql
|
337
|
+
- lib/que/scheduler/migrations/5/up.sql
|
338
|
+
- lib/que/scheduler/migrations/6/down.sql
|
339
|
+
- lib/que/scheduler/migrations/6/up.sql
|
335
340
|
- lib/que/scheduler/schedule.rb
|
336
341
|
- lib/que/scheduler/scheduler_job.rb
|
337
342
|
- lib/que/scheduler/scheduler_job_args.rb
|
@@ -348,7 +353,7 @@ metadata:
|
|
348
353
|
changelog_uri: https://github.com/hlascelles/que-scheduler/blob/master/CHANGELOG.md
|
349
354
|
source_code_uri: https://github.com/hlascelles/que-scheduler/
|
350
355
|
bug_tracker_uri: https://github.com/hlascelles/que-scheduler/issues
|
351
|
-
post_install_message:
|
356
|
+
post_install_message:
|
352
357
|
rdoc_options: []
|
353
358
|
require_paths:
|
354
359
|
- lib
|
@@ -364,7 +369,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
364
369
|
version: '0'
|
365
370
|
requirements: []
|
366
371
|
rubygems_version: 3.0.3
|
367
|
-
signing_key:
|
372
|
+
signing_key:
|
368
373
|
specification_version: 4
|
369
374
|
summary: A cron scheduler for Que
|
370
375
|
test_files: []
|