que 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +0 -1
- data/README.md +85 -44
- data/Rakefile +412 -0
- data/lib/generators/que/install_generator.rb +22 -0
- data/lib/generators/que/templates/add_que.rb +9 -0
- data/lib/que.rb +55 -5
- data/lib/que/adapters/active_record.rb +9 -0
- data/lib/que/adapters/base.rb +49 -0
- data/lib/que/adapters/connection_pool.rb +14 -0
- data/lib/que/adapters/pg.rb +17 -0
- data/lib/que/adapters/sequel.rb +14 -0
- data/lib/que/job.rb +128 -149
- data/lib/que/railtie.rb +20 -0
- data/lib/que/rake_tasks.rb +35 -0
- data/lib/que/sql.rb +121 -0
- data/lib/que/version.rb +1 -1
- data/lib/que/worker.rb +93 -156
- data/que.gemspec +8 -6
- data/spec/adapters/active_record_spec.rb +39 -0
- data/spec/adapters/connection_pool_spec.rb +12 -0
- data/spec/adapters/pg_spec.rb +5 -0
- data/spec/adapters/sequel_spec.rb +25 -0
- data/spec/connection_spec.rb +12 -0
- data/spec/helper_spec.rb +19 -0
- data/spec/pool_spec.rb +116 -0
- data/spec/queue_spec.rb +134 -0
- data/spec/spec_helper.rb +48 -25
- data/spec/support/helpers.rb +9 -0
- data/spec/support/jobs.rb +33 -0
- data/spec/support/shared_examples/adapter.rb +16 -0
- data/spec/support/shared_examples/multithreaded_adapter.rb +42 -0
- data/spec/work_spec.rb +247 -0
- data/spec/worker_spec.rb +117 -0
- metadata +73 -15
- data/spec/unit/error_spec.rb +0 -45
- data/spec/unit/queue_spec.rb +0 -67
- data/spec/unit/work_spec.rb +0 -168
- data/spec/unit/worker_spec.rb +0 -31
data/lib/que/railtie.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
module Que
|
2
|
+
class Railtie < Rails::Railtie
|
3
|
+
config.que = Que
|
4
|
+
config.que.connection = ::ActiveRecord if defined?(::ActiveRecord)
|
5
|
+
config.que.mode = :sync if Rails.env.test?
|
6
|
+
|
7
|
+
rake_tasks do
|
8
|
+
load 'que/rake_tasks.rb'
|
9
|
+
end
|
10
|
+
|
11
|
+
initializer "que.setup" do
|
12
|
+
ActiveSupport.on_load(:after_initialize) do
|
13
|
+
Que.logger ||= Rails.logger
|
14
|
+
|
15
|
+
# Only start up the worker pool if running as a server.
|
16
|
+
Que.mode ||= :async if defined? Rails::Server
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
namespace :que do
|
4
|
+
desc "Process Que's jobs using a worker pool"
|
5
|
+
task :work => :environment do
|
6
|
+
Que.logger = Logger.new(STDOUT)
|
7
|
+
Que.mode = :async
|
8
|
+
Que.worker_count = (ENV['WORKER_COUNT'] || 4).to_i
|
9
|
+
|
10
|
+
%w(INT TERM).each do |signal|
|
11
|
+
trap signal do
|
12
|
+
puts "SIG#{signal} caught, finishing current jobs and shutting down..."
|
13
|
+
Que.mode = :off
|
14
|
+
$stop = true
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
loop { sleep 0.01; break if $stop }
|
19
|
+
end
|
20
|
+
|
21
|
+
desc "Create Que's job table"
|
22
|
+
task :create => :environment do
|
23
|
+
Que.create!
|
24
|
+
end
|
25
|
+
|
26
|
+
desc "Drop Que's job table"
|
27
|
+
task :drop => :environment do
|
28
|
+
Que.drop!
|
29
|
+
end
|
30
|
+
|
31
|
+
desc "Clear Que's job table"
|
32
|
+
task :clear => :environment do
|
33
|
+
Que.clear!
|
34
|
+
end
|
35
|
+
end
|
data/lib/que/sql.rb
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
module Que
|
2
|
+
SQL = {
|
3
|
+
# Thanks to RhodiumToad in #postgresql for the job lock CTE and its lateral
|
4
|
+
# variant. They were modified only slightly from his design.
|
5
|
+
:lock_job => (
|
6
|
+
<<-SQL
|
7
|
+
WITH RECURSIVE cte AS (
|
8
|
+
SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
|
9
|
+
FROM (
|
10
|
+
SELECT job
|
11
|
+
FROM que_jobs AS job
|
12
|
+
WHERE run_at <= now()
|
13
|
+
ORDER BY priority, run_at, job_id
|
14
|
+
LIMIT 1
|
15
|
+
) AS t1
|
16
|
+
UNION ALL (
|
17
|
+
SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
|
18
|
+
FROM (
|
19
|
+
SELECT (
|
20
|
+
SELECT job
|
21
|
+
FROM que_jobs AS job
|
22
|
+
WHERE run_at <= now() AND (priority, run_at, job_id) > (cte.priority, cte.run_at, cte.job_id)
|
23
|
+
ORDER BY priority, run_at, job_id
|
24
|
+
LIMIT 1
|
25
|
+
) AS job
|
26
|
+
FROM cte
|
27
|
+
WHERE NOT cte.locked
|
28
|
+
LIMIT 1
|
29
|
+
) AS t1
|
30
|
+
)
|
31
|
+
)
|
32
|
+
SELECT job_id, priority, run_at, args, job_class, error_count
|
33
|
+
FROM cte
|
34
|
+
WHERE locked
|
35
|
+
LIMIT 1
|
36
|
+
SQL
|
37
|
+
).freeze,
|
38
|
+
|
39
|
+
# Here's an alternate scheme using LATERAL, which will work in Postgres 9.3+.
|
40
|
+
# Basically the same, but benchmark to see if it's faster/just as reliable.
|
41
|
+
|
42
|
+
# WITH RECURSIVE cte AS (
|
43
|
+
# SELECT *, pg_try_advisory_lock(s.job_id) AS locked
|
44
|
+
# FROM (
|
45
|
+
# SELECT *
|
46
|
+
# FROM que_jobs
|
47
|
+
# WHERE run_at <= now()
|
48
|
+
# ORDER BY priority, run_at, job_id
|
49
|
+
# LIMIT 1
|
50
|
+
# ) s
|
51
|
+
# UNION ALL (
|
52
|
+
# SELECT j.*, pg_try_advisory_lock(j.job_id) AS locked
|
53
|
+
# FROM (
|
54
|
+
# SELECT *
|
55
|
+
# FROM cte
|
56
|
+
# WHERE NOT locked
|
57
|
+
# ) t,
|
58
|
+
# LATERAL (
|
59
|
+
# SELECT *
|
60
|
+
# FROM que_jobs
|
61
|
+
# WHERE run_at <= now()
|
62
|
+
# AND (priority, run_at, job_id) > (t.priority, t.run_at, t.job_id)
|
63
|
+
# ORDER BY priority, run_at, job_id
|
64
|
+
# LIMIT 1
|
65
|
+
# ) j
|
66
|
+
# )
|
67
|
+
# )
|
68
|
+
# SELECT *
|
69
|
+
# FROM cte
|
70
|
+
# WHERE locked
|
71
|
+
# LIMIT 1
|
72
|
+
|
73
|
+
:create_table => (
|
74
|
+
<<-SQL
|
75
|
+
CREATE TABLE que_jobs
|
76
|
+
(
|
77
|
+
priority integer NOT NULL DEFAULT 1,
|
78
|
+
run_at timestamptz NOT NULL DEFAULT now(),
|
79
|
+
job_id bigserial NOT NULL,
|
80
|
+
job_class text NOT NULL,
|
81
|
+
args json NOT NULL DEFAULT '[]'::json,
|
82
|
+
error_count integer NOT NULL DEFAULT 0,
|
83
|
+
last_error text,
|
84
|
+
|
85
|
+
CONSTRAINT que_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
|
86
|
+
)
|
87
|
+
SQL
|
88
|
+
).freeze,
|
89
|
+
|
90
|
+
:check_job => (
|
91
|
+
<<-SQL
|
92
|
+
SELECT 1 AS one
|
93
|
+
FROM que_jobs
|
94
|
+
WHERE priority = $1::integer
|
95
|
+
AND run_at = $2::timestamptz
|
96
|
+
AND job_id = $3::bigint
|
97
|
+
SQL
|
98
|
+
).freeze,
|
99
|
+
|
100
|
+
:set_error => (
|
101
|
+
<<-SQL
|
102
|
+
UPDATE que_jobs
|
103
|
+
SET error_count = $1::integer,
|
104
|
+
run_at = $2::timestamptz,
|
105
|
+
last_error = $3::text
|
106
|
+
WHERE priority = $4::integer
|
107
|
+
AND run_at = $5::timestamptz
|
108
|
+
AND job_id = $6::bigint
|
109
|
+
SQL
|
110
|
+
).freeze,
|
111
|
+
|
112
|
+
:destroy_job => (
|
113
|
+
<<-SQL
|
114
|
+
DELETE FROM que_jobs
|
115
|
+
WHERE priority = $1::integer
|
116
|
+
AND run_at = $2::timestamptz
|
117
|
+
AND job_id = $3::bigint
|
118
|
+
SQL
|
119
|
+
).freeze
|
120
|
+
}
|
121
|
+
end
|
data/lib/que/version.rb
CHANGED
data/lib/que/worker.rb
CHANGED
@@ -1,215 +1,152 @@
|
|
1
|
-
|
2
|
-
# There are multiple workers running at a given time, each with a given
|
3
|
-
# minimum priority that they care about for jobs. A worker will continuously
|
4
|
-
# look for jobs and work them. If there's no job available, the worker will go
|
5
|
-
# to sleep until it is awakened by an external thread.
|
6
|
-
|
7
|
-
# Use Worker.state = ... to set the current state. There are three states:
|
8
|
-
# :async => Work jobs in dedicated threads. Used in production.
|
9
|
-
# :sync => Work jobs immediately, as they're queued, in the current thread. Used in testing.
|
10
|
-
# :off => Don't work jobs at all. Must use Job#work or Job.work explicitly.
|
11
|
-
|
12
|
-
# Worker.wake! will wake up the sleeping worker with the lowest minimum job
|
13
|
-
# priority. Worker.wake! may be run by another thread handling a web request,
|
14
|
-
# or by the wrangler thread (which wakes a worker every five seconds, to
|
15
|
-
# handle scheduled jobs). It only has an effect when running in async mode.
|
1
|
+
require 'monitor'
|
16
2
|
|
3
|
+
module Que
|
17
4
|
class Worker
|
18
|
-
# Each worker has a
|
19
|
-
#
|
20
|
-
#
|
21
|
-
#
|
22
|
-
include MonitorMixin
|
23
|
-
|
24
|
-
# The Worker class itself needs to be protected as well, to make sure that
|
25
|
-
# multiple threads aren't stopping/starting it at the same time.
|
26
|
-
extend MonitorMixin
|
5
|
+
# Each worker has a thread that does the actual work of running jobs.
|
6
|
+
# Since both the worker's thread and whatever thread is managing the
|
7
|
+
# worker are capable of affecting the state of the worker's thread, we
|
8
|
+
# need to synchronize access to it.
|
27
9
|
|
28
|
-
|
29
|
-
# workers should equal number of processors available to us.
|
30
|
-
PRIORITIES = [5, 5, 5, 5, 4, 3, 2, 1].freeze
|
31
|
-
|
32
|
-
# Which errors should signal a worker that it should hold off before trying
|
33
|
-
# to grab another job, in order to avoid spamming the logs.
|
34
|
-
DELAYABLE_ERRORS = %w(
|
35
|
-
Sequel::DatabaseConnectionError
|
36
|
-
Sequel::DatabaseDisconnectError
|
37
|
-
)
|
38
|
-
|
39
|
-
# How long the wrangler thread should wait between pings of the database.
|
40
|
-
# Future directions: when we have multiple dynos, add rand() to this value
|
41
|
-
# in the wrangler loop below, so that the dynos' checks will be spaced out.
|
42
|
-
SLEEP_PERIOD = 5
|
43
|
-
|
44
|
-
# How long to sleep, in repeated increments, for something to happen.
|
45
|
-
WAIT_PERIOD = 0.0001 # 0.1 ms
|
46
|
-
|
47
|
-
# How long a worker should wait before trying to get another job, in the
|
48
|
-
# event of a database connection problem.
|
49
|
-
ERROR_PERIOD = 5
|
10
|
+
include MonitorMixin
|
50
11
|
|
51
|
-
attr_reader :thread
|
12
|
+
attr_reader :thread
|
52
13
|
|
53
|
-
def initialize
|
54
|
-
super
|
14
|
+
def initialize
|
15
|
+
super # For MonitorMixin.
|
55
16
|
|
56
|
-
#
|
57
|
-
#
|
58
|
-
#
|
59
|
-
# state is set.
|
17
|
+
# We have to make sure the thread doesn't actually start the work loop
|
18
|
+
# until it has a state and directive already set up, so use a queue to
|
19
|
+
# temporarily block it.
|
60
20
|
q = Queue.new
|
61
21
|
|
62
|
-
@priority = priority
|
63
22
|
@thread = Thread.new do
|
64
23
|
q.pop
|
65
|
-
job = nil
|
66
24
|
|
67
25
|
loop do
|
68
|
-
|
26
|
+
job = Job.work
|
27
|
+
|
28
|
+
# Grab the lock and figure out what we should do next.
|
29
|
+
synchronize do
|
30
|
+
if @thread[:directive] == :stop
|
31
|
+
@thread[:state] = :stopping
|
32
|
+
elsif not job
|
33
|
+
# No work, go to sleep.
|
34
|
+
@thread[:state] = :sleeping
|
35
|
+
end
|
36
|
+
end
|
69
37
|
|
70
|
-
if @thread[:
|
71
|
-
@thread[:state] = :sleeping
|
38
|
+
if @thread[:state] == :sleeping
|
72
39
|
sleep
|
40
|
+
|
41
|
+
# Now that we're woken up, grab the lock figure out if we're stopping.
|
42
|
+
synchronize do
|
43
|
+
@thread[:state] = :stopping if @thread[:directive] == :stop
|
44
|
+
end
|
73
45
|
end
|
46
|
+
|
47
|
+
break if @thread[:state] == :stopping
|
74
48
|
end
|
75
49
|
end
|
76
50
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
q.push nil
|
51
|
+
synchronize do
|
52
|
+
@thread[:directive] = :work
|
53
|
+
@thread[:state] = :working
|
54
|
+
end
|
82
55
|
|
83
|
-
|
84
|
-
# than threads that are handling requests.
|
85
|
-
@thread.priority = -1
|
56
|
+
q.push :go!
|
86
57
|
end
|
87
58
|
|
88
|
-
|
89
|
-
# awake, does nothing and returns falsy.
|
90
|
-
def wake!
|
59
|
+
def sleeping?
|
91
60
|
synchronize do
|
92
|
-
if sleeping
|
93
|
-
# There's a very
|
94
|
-
# as sleeping
|
95
|
-
|
96
|
-
@thread
|
97
|
-
|
98
|
-
# Have to set state here so that another poke immediately after this
|
99
|
-
# one doesn't see the current state as sleeping.
|
100
|
-
@thread[:state] = :working
|
101
|
-
|
102
|
-
# Now it's safe to wake up the worker.
|
103
|
-
@thread.wakeup
|
61
|
+
if @thread[:state] == :sleeping
|
62
|
+
# There's a very small period of time between when the Worker marks
|
63
|
+
# itself as sleeping and when it actually goes to sleep. Only report
|
64
|
+
# true when we're certain the thread is sleeping.
|
65
|
+
sleep 0.0001 until @thread.status == 'sleep'
|
66
|
+
true
|
104
67
|
end
|
105
68
|
end
|
106
69
|
end
|
107
70
|
|
108
|
-
def
|
109
|
-
synchronize
|
71
|
+
def working?
|
72
|
+
synchronize do
|
73
|
+
@thread[:state] == :working
|
74
|
+
end
|
110
75
|
end
|
111
76
|
|
112
|
-
def
|
77
|
+
def wake!
|
113
78
|
synchronize do
|
114
|
-
|
115
|
-
|
116
|
-
|
79
|
+
if sleeping?
|
80
|
+
# Have to set the state here so that another thread checking
|
81
|
+
# immediately after this won't see the worker as asleep.
|
82
|
+
@thread[:state] = :working
|
83
|
+
@thread.wakeup
|
84
|
+
true
|
85
|
+
end
|
117
86
|
end
|
118
87
|
end
|
119
88
|
|
120
|
-
|
121
|
-
|
89
|
+
# This has to be called when trapping a SIGTERM, so it can't lock the monitor.
|
90
|
+
def stop!
|
91
|
+
@thread[:directive] = :stop
|
92
|
+
@thread.wakeup
|
122
93
|
end
|
123
94
|
|
124
|
-
|
125
|
-
|
126
|
-
def work_job
|
127
|
-
Job.work(:priority => priority)
|
128
|
-
rescue => error
|
129
|
-
self.class.notify_error "Worker error!", error
|
130
|
-
sleep ERROR_PERIOD if DELAYABLE_ERRORS.include?(error.class.to_s)
|
131
|
-
return true # There's work available.
|
95
|
+
def wait_until_stopped
|
96
|
+
@thread.join
|
132
97
|
end
|
133
98
|
|
134
|
-
|
135
|
-
@thread[:state] == :sleeping
|
136
|
-
end
|
99
|
+
private
|
137
100
|
|
138
|
-
|
139
|
-
|
140
|
-
|
101
|
+
# Defaults for the Worker pool.
|
102
|
+
@worker_count = 0
|
103
|
+
@sleep_period = 5
|
141
104
|
|
142
|
-
# The Worker class is responsible for managing the worker instances.
|
143
105
|
class << self
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
@wrangler ||= Thread.new { loop { wrangle } }
|
154
|
-
when :sync, :off
|
155
|
-
# Put all the workers to sleep.
|
156
|
-
workers.each(&:sleep!).each(&:wait_for_sleep)
|
157
|
-
else
|
158
|
-
raise "Bad Worker state! #{state.inspect}"
|
159
|
-
end
|
160
|
-
|
161
|
-
Que.logger.info "Set Worker to #{state}" if Que.logger
|
162
|
-
@state = state
|
106
|
+
attr_reader :mode, :sleep_period, :worker_count
|
107
|
+
|
108
|
+
def mode=(mode)
|
109
|
+
case mode
|
110
|
+
when :async
|
111
|
+
wrangler # Make sure the wrangler thread is initialized.
|
112
|
+
self.worker_count = 4
|
113
|
+
else
|
114
|
+
self.worker_count = 0
|
163
115
|
end
|
164
|
-
end
|
165
116
|
|
166
|
-
|
167
|
-
|
117
|
+
@mode = mode
|
118
|
+
Que.log :info, "Set mode to #{mode.inspect}"
|
168
119
|
end
|
169
120
|
|
170
|
-
def
|
171
|
-
|
121
|
+
def workers
|
122
|
+
@workers ||= []
|
172
123
|
end
|
173
124
|
|
174
|
-
|
175
|
-
|
176
|
-
|
125
|
+
def worker_count=(count)
|
126
|
+
if count > workers.count
|
127
|
+
(count - workers.count).times { workers << new }
|
128
|
+
elsif count < workers.count
|
129
|
+
workers.pop(workers.count - count).each(&:stop!).each(&:wait_until_stopped)
|
130
|
+
end
|
177
131
|
end
|
178
132
|
|
179
|
-
def
|
180
|
-
@
|
133
|
+
def sleep_period=(period)
|
134
|
+
@sleep_period = period
|
135
|
+
wrangler.wakeup if period
|
181
136
|
end
|
182
137
|
|
183
|
-
# Wake up just one worker to work a job, if running async.
|
184
138
|
def wake!
|
185
|
-
|
139
|
+
workers.find &:wake!
|
186
140
|
end
|
187
141
|
|
188
|
-
def
|
189
|
-
|
190
|
-
#ExceptionNotifier.notify_exception(error)
|
191
|
-
rescue => error
|
192
|
-
log_error "Error notification error!", error
|
142
|
+
def wake_all!
|
143
|
+
workers.each &:wake!
|
193
144
|
end
|
194
145
|
|
195
146
|
private
|
196
147
|
|
197
|
-
|
198
|
-
|
199
|
-
sleep SLEEP_PERIOD
|
200
|
-
wake!
|
201
|
-
rescue => error
|
202
|
-
notify_error "Wrangler Error!", error
|
203
|
-
end
|
204
|
-
|
205
|
-
def log_error(message, error)
|
206
|
-
if Que.logger
|
207
|
-
Que.logger.error <<-ERROR
|
208
|
-
#{message}
|
209
|
-
#{error.message}
|
210
|
-
#{error.backtrace.join("\n")}
|
211
|
-
ERROR
|
212
|
-
end
|
148
|
+
def wrangler
|
149
|
+
@wrangler ||= Thread.new { loop { sleep(*sleep_period); wake! } }
|
213
150
|
end
|
214
151
|
end
|
215
152
|
end
|