que-scheduler 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +10 -7
- data/lib/que/scheduler/schedule_parser.rb +32 -18
- data/lib/que/scheduler/scheduler_job.rb +18 -19
- data/lib/que/scheduler/scheduler_job_args.rb +28 -0
- data/lib/que/scheduler/version.rb +1 -1
- metadata +15 -126
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e9d071dc4b8b0357483a4a8aa5aa0b48b95e9d5
|
4
|
+
data.tar.gz: 8b8090e1eb7d1106a3a861d6c3ae2361a58a3eaf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 222d36133bf344092ad2ddff06031eecab82ee522f0da8da89d9f659f604b0770601acf676ba895003336a8076ef853538c554743f99db7571ac392e56548b92
|
7
|
+
data.tar.gz: 7cc612d46143d4fe9338e4d534c2b796c85692e2be77ed9204e02e8063e1430b6e07c5d40ee9624d93da2bcb5aa9f379911fd6fb52ae76aa70e264a6b01bddae
|
data/README.md
CHANGED
@@ -1,16 +1,19 @@
|
|
1
1
|
que-scheduler
|
2
2
|
================
|
3
3
|
|
4
|
+
[](https://badge.fury.io/rb/que-scheduler)
|
4
5
|
[](https://travis-ci.org/hlascelles/que-scheduler)
|
6
|
+
[](https://gemnasium.com/github.com/hlascelles/que-scheduler)
|
7
|
+
[](https://coveralls.io/github/hlascelles/que-scheduler?branch=master)
|
5
8
|
[](https://codeclimate.com/github/hlascelles/que-scheduler/maintainability)
|
6
9
|
|
7
|
-
|
10
|
+
## Description
|
8
11
|
|
9
12
|
que-scheduler is an extension to [Que](https://github.com/chanks/que) that adds support for scheduling
|
10
13
|
items using a cron style configuration file. It works by running as a que job itself, determining what
|
11
14
|
needs to be run, enqueueing those jobs, then enqueueing itself to check again later.
|
12
15
|
|
13
|
-
|
16
|
+
## Installation
|
14
17
|
|
15
18
|
1. To install, add the gem to your Gemfile:
|
16
19
|
```ruby
|
@@ -26,7 +29,7 @@ files, but with additional features.
|
|
26
29
|
Que::Scheduler::SchedulerJob.enqueue
|
27
30
|
```
|
28
31
|
|
29
|
-
|
32
|
+
## Schedule configuration
|
30
33
|
|
31
34
|
The schedule file is a list of que job classes with arguments and a schedule frequency (in crontab
|
32
35
|
syntax). The format is a superset of the resque-scheduler config format, so it they can be used
|
@@ -63,13 +66,13 @@ DailyBatchReport:
|
|
63
66
|
unmissable: true
|
64
67
|
```
|
65
68
|
|
66
|
-
|
69
|
+
## Environment Variables
|
67
70
|
|
68
71
|
You can configure some aspects of the gem with environment variables.
|
69
72
|
|
70
73
|
* `QUE_SCHEDULER_CONFIG_LOCATION` - The location of the schedule configuration (default `config/que_schedule.yml`)
|
71
74
|
|
72
|
-
|
75
|
+
## Redundancy and Fail-Over
|
73
76
|
|
74
77
|
Because of the way que-scheduler works, it requires no additional processes. It is, itself, a Que job.
|
75
78
|
As long as there are Que workers functioning, then jobs will continue to be scheduled correctly. There
|
@@ -77,7 +80,7 @@ are no HA concerns to worry about and no namespace collisions between different
|
|
77
80
|
|
78
81
|
Additionally, like Que, when your database is backed up, your scheduling state is stored too.
|
79
82
|
|
80
|
-
|
83
|
+
## How it works
|
81
84
|
|
82
85
|
que-scheduler is a job that reads a config file, enqueues any jobs it determines that need to be run,
|
83
86
|
then reschedules itself. The flow is as follows:
|
@@ -87,6 +90,6 @@ then reschedules itself. The flow is as follows:
|
|
87
90
|
1. Some time later it runs again. It knows what jobs it should be monitoring, and notices that some have are due. It enqueues those jobs and then itself. Repeat.
|
88
91
|
1. After a deploy that changes the config, the job notices any new jobs to schedule, and knows which ones to forget. It does not need to be re-enqueued or restarted.
|
89
92
|
|
90
|
-
|
93
|
+
## Thanks
|
91
94
|
|
92
95
|
This gem was inspired by the makers of the excellent [Que](https://github.com/chanks/que) job scheduler gem.
|
@@ -6,7 +6,7 @@ module Que
|
|
6
6
|
|
7
7
|
class ScheduleParser
|
8
8
|
class << self
|
9
|
-
def parse(
|
9
|
+
def parse(scheduler_config, scheduler_job_args)
|
10
10
|
missed_jobs = {}
|
11
11
|
schedule_dictionary = []
|
12
12
|
|
@@ -14,12 +14,18 @@ module Que
|
|
14
14
|
# new. Otherwise, check how many times we have missed the job since the last run time.
|
15
15
|
# If it is "unmissable" then we schedule all of them, with the missed time as an arg,
|
16
16
|
# otherwise just schedule it once.
|
17
|
-
|
18
|
-
|
17
|
+
scheduler_config.each do |desc|
|
18
|
+
job_name = desc[:name]
|
19
|
+
schedule_dictionary << job_name
|
20
|
+
|
21
|
+
# If we have never seen this job before, we don't want to scheduled any jobs for it.
|
22
|
+
# But we have added it to the dictionary, so it will be used to enqueue jobs next time.
|
23
|
+
next unless scheduler_job_args.job_dictionary.include?(job_name)
|
19
24
|
|
20
|
-
next unless known_jobs.include?(desc[:name])
|
21
25
|
# This has been seen before. We should check if we have missed any executions.
|
22
|
-
missed = calculate_missed_runs(
|
26
|
+
missed = calculate_missed_runs(
|
27
|
+
desc, scheduler_job_args.last_run_time, scheduler_job_args.as_time
|
28
|
+
)
|
23
29
|
missed_jobs[desc[:clazz]] = missed unless missed.empty?
|
24
30
|
end
|
25
31
|
|
@@ -30,25 +36,15 @@ module Que
|
|
30
36
|
|
31
37
|
# Given a job description, the last scheduler run time, and this run time, return all
|
32
38
|
# the instances that should be enqueued for that job class.
|
33
|
-
def calculate_missed_runs(desc,
|
34
|
-
jobs_for_class = []
|
39
|
+
def calculate_missed_runs(desc, last_run_time, as_time)
|
35
40
|
missed_times = []
|
36
|
-
last_time =
|
41
|
+
last_time = last_run_time
|
37
42
|
while (next_run = next_run_time(desc[:cron], last_time, as_time))
|
38
43
|
missed_times << next_run
|
39
44
|
last_time = next_run
|
40
45
|
end
|
41
46
|
|
42
|
-
|
43
|
-
if desc[:unmissable]
|
44
|
-
missed_times.each do |time_missed|
|
45
|
-
jobs_for_class << [time_missed] + desc[:args]
|
46
|
-
end
|
47
|
-
else
|
48
|
-
jobs_for_class << desc[:args]
|
49
|
-
end
|
50
|
-
end
|
51
|
-
jobs_for_class
|
47
|
+
generate_required_jobs_list(desc, missed_times)
|
52
48
|
end
|
53
49
|
|
54
50
|
# Given a cron, and a "last time", return the next Time the event will occur, or nil if it
|
@@ -59,6 +55,24 @@ module Que
|
|
59
55
|
next_run = next_time.to_local_time.in_time_zone(next_time.zone)
|
60
56
|
next_run <= to ? next_run : nil
|
61
57
|
end
|
58
|
+
|
59
|
+
# Given a job description, and the timestamps of the missed events, generate a list of jobs
|
60
|
+
# that can be enqueued as an array of arrays of args.
|
61
|
+
def generate_required_jobs_list(desc, missed_times)
|
62
|
+
jobs_for_class = []
|
63
|
+
unless missed_times.empty?
|
64
|
+
job_args = desc[:args]
|
65
|
+
|
66
|
+
if desc[:unmissable]
|
67
|
+
missed_times.each do |time_missed|
|
68
|
+
jobs_for_class << [time_missed] + job_args
|
69
|
+
end
|
70
|
+
else
|
71
|
+
jobs_for_class << job_args
|
72
|
+
end
|
73
|
+
end
|
74
|
+
jobs_for_class
|
75
|
+
end
|
62
76
|
end
|
63
77
|
end
|
64
78
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'que'
|
2
2
|
require 'yaml'
|
3
3
|
require_relative 'schedule_parser'
|
4
|
+
require_relative 'scheduler_job_args'
|
4
5
|
|
5
6
|
module Que
|
6
7
|
module Scheduler
|
@@ -10,41 +11,39 @@ module Que
|
|
10
11
|
# Highest possible priority.
|
11
12
|
@priority = 0
|
12
13
|
|
13
|
-
def run(
|
14
|
-
|
15
|
-
|
16
|
-
as_time = Time.zone.now
|
17
|
-
|
18
|
-
Que.log({ message: "que-scheduler last ran at #{last_time}." })
|
14
|
+
def run(options = nil, oldarg = nil)
|
15
|
+
# Early versions took separate args. We now just pass in a hash.
|
16
|
+
options = { last_run_time: options, job_dictionary: oldarg } if oldarg.present?
|
19
17
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
enqueue_self_again(
|
18
|
+
::ActiveRecord::Base.transaction do
|
19
|
+
scheduler_job_args = SchedulerJobArgs.prepare_scheduler_job_args(options)
|
20
|
+
Que.log(message: "que-scheduler last ran at #{scheduler_job_args.last_run_time}.")
|
21
|
+
result = enqueue_required_jobs(scheduler_job_args)
|
22
|
+
enqueue_self_again(scheduler_job_args, result.schedule_dictionary)
|
25
23
|
destroy
|
26
24
|
end
|
27
25
|
end
|
28
26
|
|
29
27
|
private
|
30
28
|
|
31
|
-
def enqueue_required_jobs(
|
32
|
-
|
33
|
-
|
29
|
+
def enqueue_required_jobs(scheduler_job_args)
|
30
|
+
# Obtain the hash of missed jobs. Keys are the job classes, and the values are arrays
|
31
|
+
# each containing more arrays for the arguments of that instance.
|
32
|
+
result = ScheduleParser.parse(SchedulerJob.scheduler_config, scheduler_job_args)
|
34
33
|
result.missed_jobs.each do |job_class, args_arrays|
|
35
34
|
args_arrays.each do |args|
|
36
|
-
Que.log(
|
35
|
+
Que.log(message: "que-scheduler enqueueing #{job_class} with args: #{args}")
|
37
36
|
job_class.enqueue(*args)
|
38
37
|
end
|
39
38
|
end
|
40
39
|
result
|
41
40
|
end
|
42
41
|
|
43
|
-
def enqueue_self_again(
|
42
|
+
def enqueue_self_again(scheduler_job_args, new_job_dictionary)
|
44
43
|
SchedulerJob.enqueue(
|
45
|
-
as_time,
|
46
|
-
|
47
|
-
run_at: as_time.beginning_of_minute + SCHEDULER_FREQUENCY
|
44
|
+
last_run_time: scheduler_job_args.as_time.iso8601,
|
45
|
+
job_dictionary: new_job_dictionary,
|
46
|
+
run_at: scheduler_job_args.as_time.beginning_of_minute + SCHEDULER_FREQUENCY
|
48
47
|
)
|
49
48
|
end
|
50
49
|
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'hashie'
|
2
|
+
|
3
|
+
module Que
|
4
|
+
module Scheduler
|
5
|
+
class SchedulerJobArgs < Hashie::Dash
|
6
|
+
property :last_run_time, required: true
|
7
|
+
property :job_dictionary, required: true
|
8
|
+
property :as_time, required: true
|
9
|
+
|
10
|
+
def self.prepare_scheduler_job_args(options)
|
11
|
+
parsed =
|
12
|
+
if options.nil?
|
13
|
+
# First ever run
|
14
|
+
{
|
15
|
+
last_run_time: Time.zone.now,
|
16
|
+
job_dictionary: []
|
17
|
+
}
|
18
|
+
else
|
19
|
+
{
|
20
|
+
last_run_time: Time.zone.parse(options.fetch(:last_run_time)),
|
21
|
+
job_dictionary: options.fetch(:job_dictionary)
|
22
|
+
}
|
23
|
+
end
|
24
|
+
SchedulerJobArgs.new(parsed.merge(as_time: Time.zone.now))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que-scheduler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Harry Lascelles
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -16,168 +16,56 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '4.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '4.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: fugit
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '1
|
33
|
+
version: '1'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '1
|
40
|
+
version: '1'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: hashie
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '3'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: activesupport
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - ">="
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '4.0'
|
62
|
-
type: :development
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
65
|
-
requirements:
|
66
|
-
- - ">="
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: '4.0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: bundler
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - "~>"
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '1.15'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - "~>"
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '1.15'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: pry-byebug
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - "~>"
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '3.0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - "~>"
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '3.0'
|
54
|
+
version: '3'
|
97
55
|
- !ruby/object:Gem::Dependency
|
98
|
-
name: que
|
99
|
-
requirement: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - "~>"
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: '0.1'
|
104
|
-
type: :development
|
105
|
-
prerelease: false
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
107
|
-
requirements:
|
108
|
-
- - "~>"
|
109
|
-
- !ruby/object:Gem::Version
|
110
|
-
version: '0.1'
|
111
|
-
- !ruby/object:Gem::Dependency
|
112
|
-
name: rake
|
113
|
-
requirement: !ruby/object:Gem::Requirement
|
114
|
-
requirements:
|
115
|
-
- - "~>"
|
116
|
-
- !ruby/object:Gem::Version
|
117
|
-
version: '10.0'
|
118
|
-
type: :development
|
119
|
-
prerelease: false
|
120
|
-
version_requirements: !ruby/object:Gem::Requirement
|
121
|
-
requirements:
|
122
|
-
- - "~>"
|
123
|
-
- !ruby/object:Gem::Version
|
124
|
-
version: '10.0'
|
125
|
-
- !ruby/object:Gem::Dependency
|
126
|
-
name: rspec
|
127
|
-
requirement: !ruby/object:Gem::Requirement
|
128
|
-
requirements:
|
129
|
-
- - "~>"
|
130
|
-
- !ruby/object:Gem::Version
|
131
|
-
version: '3.0'
|
132
|
-
type: :development
|
133
|
-
prerelease: false
|
134
|
-
version_requirements: !ruby/object:Gem::Requirement
|
135
|
-
requirements:
|
136
|
-
- - "~>"
|
137
|
-
- !ruby/object:Gem::Version
|
138
|
-
version: '3.0'
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: rubocop
|
141
|
-
requirement: !ruby/object:Gem::Requirement
|
142
|
-
requirements:
|
143
|
-
- - "~>"
|
144
|
-
- !ruby/object:Gem::Version
|
145
|
-
version: '0.51'
|
146
|
-
type: :development
|
147
|
-
prerelease: false
|
148
|
-
version_requirements: !ruby/object:Gem::Requirement
|
149
|
-
requirements:
|
150
|
-
- - "~>"
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: '0.51'
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
|
-
name: timecop
|
155
|
-
requirement: !ruby/object:Gem::Requirement
|
156
|
-
requirements:
|
157
|
-
- - "~>"
|
158
|
-
- !ruby/object:Gem::Version
|
159
|
-
version: '0.7'
|
160
|
-
type: :development
|
161
|
-
prerelease: false
|
162
|
-
version_requirements: !ruby/object:Gem::Requirement
|
163
|
-
requirements:
|
164
|
-
- - "~>"
|
165
|
-
- !ruby/object:Gem::Version
|
166
|
-
version: '0.7'
|
167
|
-
- !ruby/object:Gem::Dependency
|
168
|
-
name: zonebie
|
56
|
+
name: que
|
169
57
|
requirement: !ruby/object:Gem::Requirement
|
170
58
|
requirements:
|
171
59
|
- - "~>"
|
172
60
|
- !ruby/object:Gem::Version
|
173
|
-
version: '0.
|
174
|
-
type: :
|
61
|
+
version: '0.10'
|
62
|
+
type: :runtime
|
175
63
|
prerelease: false
|
176
64
|
version_requirements: !ruby/object:Gem::Requirement
|
177
65
|
requirements:
|
178
66
|
- - "~>"
|
179
67
|
- !ruby/object:Gem::Version
|
180
|
-
version: '0.
|
68
|
+
version: '0.10'
|
181
69
|
description: A lightweight cron scheduler for the async job worker Que
|
182
70
|
email:
|
183
71
|
- harry@harrylascelles.com
|
@@ -189,8 +77,9 @@ files:
|
|
189
77
|
- lib/que/scheduler.rb
|
190
78
|
- lib/que/scheduler/schedule_parser.rb
|
191
79
|
- lib/que/scheduler/scheduler_job.rb
|
80
|
+
- lib/que/scheduler/scheduler_job_args.rb
|
192
81
|
- lib/que/scheduler/version.rb
|
193
|
-
homepage: https://
|
82
|
+
homepage: https://github.com/hlascelles/que-scheduler
|
194
83
|
licenses:
|
195
84
|
- MIT
|
196
85
|
metadata: {}
|