que 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/README.md +12 -31
- data/docs/advanced_setup.md +50 -0
- data/docs/error_handling.md +17 -0
- data/docs/inspecting_the_queue.md +100 -0
- data/docs/managing_workers.md +67 -0
- data/docs/using_plain_connections.md +34 -0
- data/docs/using_sequel.md +27 -0
- data/docs/writing_reliable_jobs.md +62 -0
- data/lib/que.rb +33 -5
- data/lib/que/adapters/base.rb +9 -1
- data/lib/que/adapters/pg.rb +2 -0
- data/lib/que/adapters/sequel.rb +4 -0
- data/lib/que/job.rb +30 -64
- data/lib/que/rake_tasks.rb +0 -1
- data/lib/que/sql.rb +119 -74
- data/lib/que/version.rb +1 -1
- data/lib/que/worker.rb +39 -26
- data/spec/adapters/active_record_spec.rb +7 -26
- data/spec/adapters/connection_pool_spec.rb +1 -2
- data/spec/adapters/pg_spec.rb +34 -0
- data/spec/adapters/sequel_spec.rb +16 -28
- data/spec/spec_helper.rb +46 -34
- data/spec/support/shared_examples/adapter.rb +16 -3
- data/spec/support/shared_examples/{multithreaded_adapter.rb → multi_threaded_adapter.rb} +3 -1
- data/spec/unit/helper_spec.rb +0 -5
- data/spec/unit/pool_spec.rb +75 -23
- data/spec/unit/queue_spec.rb +5 -1
- data/spec/unit/states_spec.rb +52 -0
- data/spec/unit/stats_spec.rb +42 -0
- data/spec/unit/work_spec.rb +15 -20
- data/spec/unit/worker_spec.rb +18 -3
- data/tasks/safe_shutdown.rb +5 -3
- metadata +16 -5
data/spec/unit/queue_spec.rb
CHANGED
@@ -3,9 +3,13 @@ require 'spec_helper'
|
|
3
3
|
describe Que::Job, '.queue' do
|
4
4
|
it "should be able to queue a job" do
|
5
5
|
DB[:que_jobs].count.should be 0
|
6
|
-
Que::Job.queue
|
6
|
+
result = Que::Job.queue
|
7
7
|
DB[:que_jobs].count.should be 1
|
8
8
|
|
9
|
+
result.should be_an_instance_of Que::Job
|
10
|
+
result.attrs[:priority].should == '1'
|
11
|
+
result.attrs[:args].should == []
|
12
|
+
|
9
13
|
job = DB[:que_jobs].first
|
10
14
|
job[:priority].should be 1
|
11
15
|
job[:run_at].should be_within(3).of Time.now
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Que, '.worker_states' do
|
4
|
+
it "should return a list of the job types in the queue, their counts and the number of each currently running" do
|
5
|
+
Que.adapter = QUE_ADAPTERS[:connection_pool]
|
6
|
+
|
7
|
+
class WorkerStateJob < BlockJob
|
8
|
+
def run
|
9
|
+
$pid = Que.execute("select pg_backend_pid()").first['pg_backend_pid']
|
10
|
+
super
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
WorkerStateJob.queue :priority => 2
|
15
|
+
|
16
|
+
# Ensure that the portion of the SQL query that accounts for bigint
|
17
|
+
# job_ids functions correctly.
|
18
|
+
DB[:que_jobs].update(:job_id => 2**33)
|
19
|
+
|
20
|
+
begin
|
21
|
+
t = Thread.new { Que::Job.work }
|
22
|
+
$q1.pop
|
23
|
+
|
24
|
+
states = Que.worker_states
|
25
|
+
states.length.should be 1
|
26
|
+
|
27
|
+
state = states.first
|
28
|
+
state.keys.should == %w(priority run_at job_id job_class args error_count last_error pg_backend_pid pg_state pg_state_changed_at pg_last_query pg_last_query_started_at pg_transaction_started_at pg_waiting_on_lock)
|
29
|
+
|
30
|
+
state[:priority].should == '2'
|
31
|
+
Time.parse(state[:run_at]).should be_within(3).of Time.now
|
32
|
+
state[:job_id].should == (2**33).to_s
|
33
|
+
state[:job_class].should == 'WorkerStateJob'
|
34
|
+
state[:args].should == '[]'
|
35
|
+
state[:error_count].should == '0'
|
36
|
+
state[:last_error].should be nil
|
37
|
+
|
38
|
+
state[:pg_backend_pid].should == $pid.to_s
|
39
|
+
state[:pg_state].should == 'idle'
|
40
|
+
Time.parse(state[:pg_state_changed_at]).should be_within(3).of Time.now
|
41
|
+
state[:pg_last_query].should == 'select pg_backend_pid()'
|
42
|
+
Time.parse(state[:pg_last_query_started_at]).should be_within(3).of Time.now
|
43
|
+
state[:pg_transaction_started_at].should == nil
|
44
|
+
state[:pg_waiting_on_lock].should == 'f'
|
45
|
+
ensure
|
46
|
+
if t
|
47
|
+
t.kill
|
48
|
+
t.join
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end if QUE_ADAPTERS[:connection_pool]
|
52
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Que, '.job_stats' do
|
4
|
+
it "should return a list of the job types in the queue, their counts and the number of each currently running" do
|
5
|
+
BlockJob.queue
|
6
|
+
Que::Job.queue
|
7
|
+
|
8
|
+
# Have to tweak the job_id to ensure that the portion of the SQL query
|
9
|
+
# that accounts for bigint job_ids functions correctly.
|
10
|
+
old = Time.now - 3600
|
11
|
+
DB[:que_jobs].where(:job_class => "Que::Job").update(:job_id => 2**33, :error_count => 5, :run_at => old)
|
12
|
+
|
13
|
+
Que::Job.queue
|
14
|
+
|
15
|
+
begin
|
16
|
+
DB.get{pg_advisory_lock(2**33)}
|
17
|
+
|
18
|
+
stats = Que.job_stats
|
19
|
+
stats.length.should == 2
|
20
|
+
|
21
|
+
qj, bj = stats
|
22
|
+
|
23
|
+
qj.keys.should == %w(job_class count count_working count_errored highest_error_count oldest_run_at)
|
24
|
+
|
25
|
+
qj[:job_class].should == 'Que::Job'
|
26
|
+
qj[:count].should == '2'
|
27
|
+
qj[:count_working].should == '1'
|
28
|
+
qj[:count_errored].should == '1'
|
29
|
+
qj[:highest_error_count].should == '5'
|
30
|
+
Time.parse(qj[:oldest_run_at]).should be_within(3).of old
|
31
|
+
|
32
|
+
bj[:job_class].should == 'BlockJob'
|
33
|
+
bj[:count].should == '1'
|
34
|
+
bj[:count_working].should == '0'
|
35
|
+
bj[:count_errored].should == '0'
|
36
|
+
bj[:highest_error_count].should == '0'
|
37
|
+
Time.parse(bj[:oldest_run_at]).should be_within(3).of Time.now
|
38
|
+
ensure
|
39
|
+
DB.get{pg_advisory_unlock_all{}}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/spec/unit/work_spec.rb
CHANGED
@@ -7,9 +7,6 @@ describe Que::Job, '.work' do
|
|
7
7
|
Que::Job.work.should be_an_instance_of ArgsJob
|
8
8
|
DB[:que_jobs].count.should be 0
|
9
9
|
$passed_args.should == [1, 'two', {'three' => 3}]
|
10
|
-
|
11
|
-
# Should clear advisory lock.
|
12
|
-
DB[:pg_locks].where(:locktype => 'advisory').should be_empty
|
13
10
|
end
|
14
11
|
|
15
12
|
it "should make a job's argument hashes indifferently accessible" do
|
@@ -20,14 +17,10 @@ describe Que::Job, '.work' do
|
|
20
17
|
DB[:que_jobs].count.should be 0
|
21
18
|
|
22
19
|
$passed_args.last[:array].first[:number].should == 3
|
23
|
-
|
24
|
-
# Should clear advisory lock.
|
25
|
-
DB[:pg_locks].where(:locktype => 'advisory').should be_empty
|
26
20
|
end
|
27
21
|
|
28
22
|
it "should not fail if there are no jobs to work" do
|
29
23
|
Que::Job.work.should be nil
|
30
|
-
DB[:pg_locks].where(:locktype => 'advisory').should be_empty
|
31
24
|
end
|
32
25
|
|
33
26
|
it "should write messages to the logger" do
|
@@ -50,16 +43,15 @@ describe Que::Job, '.work' do
|
|
50
43
|
end
|
51
44
|
|
52
45
|
it "should prefer a job with a higher priority" do
|
53
|
-
|
54
|
-
Que::Job.queue :priority =>
|
55
|
-
|
56
|
-
DB[:que_jobs].order(:job_id).select_map(:priority).should == [5, 1, 5]
|
46
|
+
# 1 is highest priority.
|
47
|
+
[5, 4, 3, 2, 1, 2, 3, 4, 5].map{|p| Que::Job.queue :priority => p}
|
48
|
+
DB[:que_jobs].order(:job_id).select_map(:priority).should == [5, 4, 3, 2, 1, 2, 3, 4, 5]
|
57
49
|
|
58
50
|
Que::Job.work.should be_an_instance_of Que::Job
|
59
|
-
DB[:que_jobs].select_map(:priority).should == [5, 5]
|
51
|
+
DB[:que_jobs].select_map(:priority).should == [5, 4, 3, 2, 2, 3, 4, 5]
|
60
52
|
end
|
61
53
|
|
62
|
-
it "should prefer a job that was scheduled to run longer ago" do
|
54
|
+
it "should prefer a job that was scheduled to run longer ago when priorities are equal" do
|
63
55
|
Que::Job.queue :run_at => Time.now - 30
|
64
56
|
Que::Job.queue :run_at => Time.now - 60
|
65
57
|
Que::Job.queue :run_at => Time.now - 30
|
@@ -70,7 +62,7 @@ describe Que::Job, '.work' do
|
|
70
62
|
DB[:que_jobs].order_by(:job_id).select_map(:run_at).should == [recent1, recent2]
|
71
63
|
end
|
72
64
|
|
73
|
-
it "should prefer a job that was queued earlier
|
65
|
+
it "should prefer a job that was queued earlier when priorities and run_ats are equal" do
|
74
66
|
run_at = Time.now - 30
|
75
67
|
Que::Job.queue :run_at => run_at
|
76
68
|
Que::Job.queue :run_at => run_at
|
@@ -100,19 +92,22 @@ describe Que::Job, '.work' do
|
|
100
92
|
thread = Thread.new { Que::Job.work }
|
101
93
|
|
102
94
|
$q1.pop
|
103
|
-
DB[:pg_locks].where(:locktype => 'advisory'
|
95
|
+
DB[:pg_locks].where(:locktype => 'advisory').select_map(:objid).should == [id]
|
104
96
|
$q2.push nil
|
105
97
|
|
106
98
|
thread.join
|
107
99
|
end
|
108
100
|
|
109
|
-
it "should
|
110
|
-
Que::Job.queue
|
111
|
-
|
101
|
+
it "should skip jobs that are advisory-locked" do
|
102
|
+
Que::Job.queue :priority => 2
|
103
|
+
Que::Job.queue :priority => 1
|
104
|
+
Que::Job.queue :priority => 3
|
105
|
+
id = DB[:que_jobs].where(:priority => 1).get(:job_id)
|
112
106
|
|
113
107
|
begin
|
114
108
|
DB.select{pg_advisory_lock(id)}.single_value
|
115
|
-
Que::Job.work.should
|
109
|
+
Que::Job.work.should be_an_instance_of Que::Job
|
110
|
+
DB[:que_jobs].order_by(:job_id).select_map(:priority).should == [1, 3]
|
116
111
|
ensure
|
117
112
|
DB.select{pg_advisory_unlock(id)}.single_value
|
118
113
|
end
|
@@ -234,7 +229,7 @@ describe Que::Job, '.work' do
|
|
234
229
|
Que::Job.work.should be false
|
235
230
|
end
|
236
231
|
|
237
|
-
it "should
|
232
|
+
it "should throw an error properly if there's no corresponding job class" do
|
238
233
|
DB[:que_jobs].insert :job_class => "NonexistentClass"
|
239
234
|
Que::Job.work.should be true
|
240
235
|
DB[:que_jobs].count.should be 1
|
data/spec/unit/worker_spec.rb
CHANGED
@@ -75,7 +75,7 @@ describe Que::Worker do
|
|
75
75
|
end
|
76
76
|
end
|
77
77
|
|
78
|
-
it "should receive and respect a notification to
|
78
|
+
it "should receive and respect a notification to stop down when it is working, after its current job completes" do
|
79
79
|
begin
|
80
80
|
BlockJob.queue :priority => 1
|
81
81
|
Que::Job.queue :priority => 5
|
@@ -100,7 +100,7 @@ describe Que::Worker do
|
|
100
100
|
end
|
101
101
|
end
|
102
102
|
|
103
|
-
it "should receive and respect a notification to
|
103
|
+
it "should receive and respect a notification to stop when it is currently asleep" do
|
104
104
|
begin
|
105
105
|
@worker = Que::Worker.new
|
106
106
|
sleep_until { @worker.sleeping? }
|
@@ -115,7 +115,7 @@ describe Que::Worker do
|
|
115
115
|
end
|
116
116
|
end
|
117
117
|
|
118
|
-
it "should receive and respect a notification to
|
118
|
+
it "should receive and respect a notification to stop immediately when it is working, and kill the job" do
|
119
119
|
begin
|
120
120
|
# Worker#stop! can leave the database connection in an unpredictable
|
121
121
|
# state, which would impact the rest of the tests, so we need a special
|
@@ -133,4 +133,19 @@ describe Que::Worker do
|
|
133
133
|
pg.close if pg
|
134
134
|
end
|
135
135
|
end
|
136
|
+
|
137
|
+
it "should receive and respect a notification to stop immediately when it is currently asleep" do
|
138
|
+
begin
|
139
|
+
@worker = Que::Worker.new
|
140
|
+
sleep_until { @worker.sleeping? }
|
141
|
+
|
142
|
+
@worker.stop!
|
143
|
+
@worker.wait_until_stopped
|
144
|
+
ensure
|
145
|
+
if @worker
|
146
|
+
@worker.thread.kill
|
147
|
+
@worker.thread.join
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
136
151
|
end
|
data/tasks/safe_shutdown.rb
CHANGED
@@ -3,13 +3,15 @@
|
|
3
3
|
# The situation we're trying to avoid occurs when the process dies while a job
|
4
4
|
# is in the middle of a transaction - ideally, the transaction would be rolled
|
5
5
|
# back and the job could just be reattempted later, but if we're not careful,
|
6
|
-
# the transaction could be committed
|
6
|
+
# the transaction could be committed prematurely. For specifics, see here:
|
7
7
|
|
8
8
|
# http://coderrr.wordpress.com/2011/05/03/beware-of-threadkill-or-your-activerecord-transactions-are-in-danger-of-being-partially-committed/
|
9
9
|
|
10
10
|
# So, this task opens a transaction within a job, makes a write, then prompts
|
11
|
-
# you to kill it with one of a few signals.
|
12
|
-
#
|
11
|
+
# you to kill it with one of a few signals. You can then run it again to make
|
12
|
+
# sure that the write was rolled back (if it wasn't, Que isn't functioning
|
13
|
+
# like it should). This task only explicitly tests Sequel, but the behavior
|
14
|
+
# for ActiveRecord is very similar.
|
13
15
|
|
14
16
|
task :safe_shutdown do
|
15
17
|
require 'sequel'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Hanks
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,6 +52,13 @@ files:
|
|
52
52
|
- LICENSE.txt
|
53
53
|
- README.md
|
54
54
|
- Rakefile
|
55
|
+
- docs/advanced_setup.md
|
56
|
+
- docs/error_handling.md
|
57
|
+
- docs/inspecting_the_queue.md
|
58
|
+
- docs/managing_workers.md
|
59
|
+
- docs/using_plain_connections.md
|
60
|
+
- docs/using_sequel.md
|
61
|
+
- docs/writing_reliable_jobs.md
|
55
62
|
- lib/generators/que/install_generator.rb
|
56
63
|
- lib/generators/que/templates/add_que.rb
|
57
64
|
- lib/que.rb
|
@@ -75,11 +82,13 @@ files:
|
|
75
82
|
- spec/support/helpers.rb
|
76
83
|
- spec/support/jobs.rb
|
77
84
|
- spec/support/shared_examples/adapter.rb
|
78
|
-
- spec/support/shared_examples/
|
85
|
+
- spec/support/shared_examples/multi_threaded_adapter.rb
|
79
86
|
- spec/unit/connection_spec.rb
|
80
87
|
- spec/unit/helper_spec.rb
|
81
88
|
- spec/unit/pool_spec.rb
|
82
89
|
- spec/unit/queue_spec.rb
|
90
|
+
- spec/unit/states_spec.rb
|
91
|
+
- spec/unit/stats_spec.rb
|
83
92
|
- spec/unit/work_spec.rb
|
84
93
|
- spec/unit/worker_spec.rb
|
85
94
|
- tasks/benchmark.rb
|
@@ -105,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
105
114
|
version: '0'
|
106
115
|
requirements: []
|
107
116
|
rubyforge_project:
|
108
|
-
rubygems_version: 2.1
|
117
|
+
rubygems_version: 2.2.0.rc.1
|
109
118
|
signing_key:
|
110
119
|
specification_version: 4
|
111
120
|
summary: A PostgreSQL-based Job Queue
|
@@ -118,10 +127,12 @@ test_files:
|
|
118
127
|
- spec/support/helpers.rb
|
119
128
|
- spec/support/jobs.rb
|
120
129
|
- spec/support/shared_examples/adapter.rb
|
121
|
-
- spec/support/shared_examples/
|
130
|
+
- spec/support/shared_examples/multi_threaded_adapter.rb
|
122
131
|
- spec/unit/connection_spec.rb
|
123
132
|
- spec/unit/helper_spec.rb
|
124
133
|
- spec/unit/pool_spec.rb
|
125
134
|
- spec/unit/queue_spec.rb
|
135
|
+
- spec/unit/states_spec.rb
|
136
|
+
- spec/unit/stats_spec.rb
|
126
137
|
- spec/unit/work_spec.rb
|
127
138
|
- spec/unit/worker_spec.rb
|