que 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,9 +3,13 @@ require 'spec_helper'
3
3
  describe Que::Job, '.queue' do
4
4
  it "should be able to queue a job" do
5
5
  DB[:que_jobs].count.should be 0
6
- Que::Job.queue
6
+ result = Que::Job.queue
7
7
  DB[:que_jobs].count.should be 1
8
8
 
9
+ result.should be_an_instance_of Que::Job
10
+ result.attrs[:priority].should == '1'
11
+ result.attrs[:args].should == []
12
+
9
13
  job = DB[:que_jobs].first
10
14
  job[:priority].should be 1
11
15
  job[:run_at].should be_within(3).of Time.now
@@ -0,0 +1,52 @@
1
+ require 'spec_helper'
2
+
3
+ describe Que, '.worker_states' do
4
+ it "should return a list of the job types in the queue, their counts and the number of each currently running" do
5
+ Que.adapter = QUE_ADAPTERS[:connection_pool]
6
+
7
+ class WorkerStateJob < BlockJob
8
+ def run
9
+ $pid = Que.execute("select pg_backend_pid()").first['pg_backend_pid']
10
+ super
11
+ end
12
+ end
13
+
14
+ WorkerStateJob.queue :priority => 2
15
+
16
+ # Ensure that the portion of the SQL query that accounts for bigint
17
+ # job_ids functions correctly.
18
+ DB[:que_jobs].update(:job_id => 2**33)
19
+
20
+ begin
21
+ t = Thread.new { Que::Job.work }
22
+ $q1.pop
23
+
24
+ states = Que.worker_states
25
+ states.length.should be 1
26
+
27
+ state = states.first
28
+ state.keys.should == %w(priority run_at job_id job_class args error_count last_error pg_backend_pid pg_state pg_state_changed_at pg_last_query pg_last_query_started_at pg_transaction_started_at pg_waiting_on_lock)
29
+
30
+ state[:priority].should == '2'
31
+ Time.parse(state[:run_at]).should be_within(3).of Time.now
32
+ state[:job_id].should == (2**33).to_s
33
+ state[:job_class].should == 'WorkerStateJob'
34
+ state[:args].should == '[]'
35
+ state[:error_count].should == '0'
36
+ state[:last_error].should be nil
37
+
38
+ state[:pg_backend_pid].should == $pid.to_s
39
+ state[:pg_state].should == 'idle'
40
+ Time.parse(state[:pg_state_changed_at]).should be_within(3).of Time.now
41
+ state[:pg_last_query].should == 'select pg_backend_pid()'
42
+ Time.parse(state[:pg_last_query_started_at]).should be_within(3).of Time.now
43
+ state[:pg_transaction_started_at].should == nil
44
+ state[:pg_waiting_on_lock].should == 'f'
45
+ ensure
46
+ if t
47
+ t.kill
48
+ t.join
49
+ end
50
+ end
51
+ end if QUE_ADAPTERS[:connection_pool]
52
+ end
@@ -0,0 +1,42 @@
1
+ require 'spec_helper'
2
+
3
+ describe Que, '.job_stats' do
4
+ it "should return a list of the job types in the queue, their counts and the number of each currently running" do
5
+ BlockJob.queue
6
+ Que::Job.queue
7
+
8
+ # Have to tweak the job_id to ensure that the portion of the SQL query
9
+ # that accounts for bigint job_ids functions correctly.
10
+ old = Time.now - 3600
11
+ DB[:que_jobs].where(:job_class => "Que::Job").update(:job_id => 2**33, :error_count => 5, :run_at => old)
12
+
13
+ Que::Job.queue
14
+
15
+ begin
16
+ DB.get{pg_advisory_lock(2**33)}
17
+
18
+ stats = Que.job_stats
19
+ stats.length.should == 2
20
+
21
+ qj, bj = stats
22
+
23
+ qj.keys.should == %w(job_class count count_working count_errored highest_error_count oldest_run_at)
24
+
25
+ qj[:job_class].should == 'Que::Job'
26
+ qj[:count].should == '2'
27
+ qj[:count_working].should == '1'
28
+ qj[:count_errored].should == '1'
29
+ qj[:highest_error_count].should == '5'
30
+ Time.parse(qj[:oldest_run_at]).should be_within(3).of old
31
+
32
+ bj[:job_class].should == 'BlockJob'
33
+ bj[:count].should == '1'
34
+ bj[:count_working].should == '0'
35
+ bj[:count_errored].should == '0'
36
+ bj[:highest_error_count].should == '0'
37
+ Time.parse(bj[:oldest_run_at]).should be_within(3).of Time.now
38
+ ensure
39
+ DB.get{pg_advisory_unlock_all{}}
40
+ end
41
+ end
42
+ end
@@ -7,9 +7,6 @@ describe Que::Job, '.work' do
7
7
  Que::Job.work.should be_an_instance_of ArgsJob
8
8
  DB[:que_jobs].count.should be 0
9
9
  $passed_args.should == [1, 'two', {'three' => 3}]
10
-
11
- # Should clear advisory lock.
12
- DB[:pg_locks].where(:locktype => 'advisory').should be_empty
13
10
  end
14
11
 
15
12
  it "should make a job's argument hashes indifferently accessible" do
@@ -20,14 +17,10 @@ describe Que::Job, '.work' do
20
17
  DB[:que_jobs].count.should be 0
21
18
 
22
19
  $passed_args.last[:array].first[:number].should == 3
23
-
24
- # Should clear advisory lock.
25
- DB[:pg_locks].where(:locktype => 'advisory').should be_empty
26
20
  end
27
21
 
28
22
  it "should not fail if there are no jobs to work" do
29
23
  Que::Job.work.should be nil
30
- DB[:pg_locks].where(:locktype => 'advisory').should be_empty
31
24
  end
32
25
 
33
26
  it "should write messages to the logger" do
@@ -50,16 +43,15 @@ describe Que::Job, '.work' do
50
43
  end
51
44
 
52
45
  it "should prefer a job with a higher priority" do
53
- Que::Job.queue :priority => 5
54
- Que::Job.queue :priority => 1
55
- Que::Job.queue :priority => 5
56
- DB[:que_jobs].order(:job_id).select_map(:priority).should == [5, 1, 5]
46
+ # 1 is highest priority.
47
+ [5, 4, 3, 2, 1, 2, 3, 4, 5].map{|p| Que::Job.queue :priority => p}
48
+ DB[:que_jobs].order(:job_id).select_map(:priority).should == [5, 4, 3, 2, 1, 2, 3, 4, 5]
57
49
 
58
50
  Que::Job.work.should be_an_instance_of Que::Job
59
- DB[:que_jobs].select_map(:priority).should == [5, 5]
51
+ DB[:que_jobs].select_map(:priority).should == [5, 4, 3, 2, 2, 3, 4, 5]
60
52
  end
61
53
 
62
- it "should prefer a job that was scheduled to run longer ago" do
54
+ it "should prefer a job that was scheduled to run longer ago when priorities are equal" do
63
55
  Que::Job.queue :run_at => Time.now - 30
64
56
  Que::Job.queue :run_at => Time.now - 60
65
57
  Que::Job.queue :run_at => Time.now - 30
@@ -70,7 +62,7 @@ describe Que::Job, '.work' do
70
62
  DB[:que_jobs].order_by(:job_id).select_map(:run_at).should == [recent1, recent2]
71
63
  end
72
64
 
73
- it "should prefer a job that was queued earlier, judging by the job_id" do
65
+ it "should prefer a job that was queued earlier when priorities and run_ats are equal" do
74
66
  run_at = Time.now - 30
75
67
  Que::Job.queue :run_at => run_at
76
68
  Que::Job.queue :run_at => run_at
@@ -100,19 +92,22 @@ describe Que::Job, '.work' do
100
92
  thread = Thread.new { Que::Job.work }
101
93
 
102
94
  $q1.pop
103
- DB[:pg_locks].where(:locktype => 'advisory', :objid => id).count.should be 1
95
+ DB[:pg_locks].where(:locktype => 'advisory').select_map(:objid).should == [id]
104
96
  $q2.push nil
105
97
 
106
98
  thread.join
107
99
  end
108
100
 
109
- it "should not work jobs that are advisory-locked" do
110
- Que::Job.queue
111
- id = DB[:que_jobs].get(:job_id)
101
+ it "should skip jobs that are advisory-locked" do
102
+ Que::Job.queue :priority => 2
103
+ Que::Job.queue :priority => 1
104
+ Que::Job.queue :priority => 3
105
+ id = DB[:que_jobs].where(:priority => 1).get(:job_id)
112
106
 
113
107
  begin
114
108
  DB.select{pg_advisory_lock(id)}.single_value
115
- Que::Job.work.should be nil
109
+ Que::Job.work.should be_an_instance_of Que::Job
110
+ DB[:que_jobs].order_by(:job_id).select_map(:priority).should == [1, 3]
116
111
  ensure
117
112
  DB.select{pg_advisory_unlock(id)}.single_value
118
113
  end
@@ -234,7 +229,7 @@ describe Que::Job, '.work' do
234
229
  Que::Job.work.should be false
235
230
  end
236
231
 
237
- it "should behave sensibly if there's no corresponding job class" do
232
+ it "should throw an error properly if there's no corresponding job class" do
238
233
  DB[:que_jobs].insert :job_class => "NonexistentClass"
239
234
  Que::Job.work.should be true
240
235
  DB[:que_jobs].count.should be 1
@@ -75,7 +75,7 @@ describe Que::Worker do
75
75
  end
76
76
  end
77
77
 
78
- it "should receive and respect a notification to shut down when it is working, after its current job completes" do
78
+ it "should receive and respect a notification to stop down when it is working, after its current job completes" do
79
79
  begin
80
80
  BlockJob.queue :priority => 1
81
81
  Que::Job.queue :priority => 5
@@ -100,7 +100,7 @@ describe Que::Worker do
100
100
  end
101
101
  end
102
102
 
103
- it "should receive and respect a notification to shut down when it is asleep" do
103
+ it "should receive and respect a notification to stop when it is currently asleep" do
104
104
  begin
105
105
  @worker = Que::Worker.new
106
106
  sleep_until { @worker.sleeping? }
@@ -115,7 +115,7 @@ describe Que::Worker do
115
115
  end
116
116
  end
117
117
 
118
- it "should receive and respect a notification to kill its job and stop running immediately" do
118
+ it "should receive and respect a notification to stop immediately when it is working, and kill the job" do
119
119
  begin
120
120
  # Worker#stop! can leave the database connection in an unpredictable
121
121
  # state, which would impact the rest of the tests, so we need a special
@@ -133,4 +133,19 @@ describe Que::Worker do
133
133
  pg.close if pg
134
134
  end
135
135
  end
136
+
137
+ it "should receive and respect a notification to stop immediately when it is currently asleep" do
138
+ begin
139
+ @worker = Que::Worker.new
140
+ sleep_until { @worker.sleeping? }
141
+
142
+ @worker.stop!
143
+ @worker.wait_until_stopped
144
+ ensure
145
+ if @worker
146
+ @worker.thread.kill
147
+ @worker.thread.join
148
+ end
149
+ end
150
+ end
136
151
  end
@@ -3,13 +3,15 @@
3
3
  # The situation we're trying to avoid occurs when the process dies while a job
4
4
  # is in the middle of a transaction - ideally, the transaction would be rolled
5
5
  # back and the job could just be reattempted later, but if we're not careful,
6
- # the transaction could be committed too early. For specifics, see this post:
6
+ # the transaction could be committed prematurely. For specifics, see here:
7
7
 
8
8
  # http://coderrr.wordpress.com/2011/05/03/beware-of-threadkill-or-your-activerecord-transactions-are-in-danger-of-being-partially-committed/
9
9
 
10
10
  # So, this task opens a transaction within a job, makes a write, then prompts
11
- # you to kill it with one of a few signals. It also checks to see whether the
12
- # previous shutdown was clean or not.
11
+ # you to kill it with one of a few signals. You can then run it again to make
12
+ # sure that the write was rolled back (if it wasn't, Que isn't functioning
13
+ # like it should). This task only explicitly tests Sequel, but the behavior
14
+ # for ActiveRecord is very similar.
13
15
 
14
16
  task :safe_shutdown do
15
17
  require 'sequel'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: que
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Hanks
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-12-22 00:00:00.000000000 Z
11
+ date: 2014-01-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -52,6 +52,13 @@ files:
52
52
  - LICENSE.txt
53
53
  - README.md
54
54
  - Rakefile
55
+ - docs/advanced_setup.md
56
+ - docs/error_handling.md
57
+ - docs/inspecting_the_queue.md
58
+ - docs/managing_workers.md
59
+ - docs/using_plain_connections.md
60
+ - docs/using_sequel.md
61
+ - docs/writing_reliable_jobs.md
55
62
  - lib/generators/que/install_generator.rb
56
63
  - lib/generators/que/templates/add_que.rb
57
64
  - lib/que.rb
@@ -75,11 +82,13 @@ files:
75
82
  - spec/support/helpers.rb
76
83
  - spec/support/jobs.rb
77
84
  - spec/support/shared_examples/adapter.rb
78
- - spec/support/shared_examples/multithreaded_adapter.rb
85
+ - spec/support/shared_examples/multi_threaded_adapter.rb
79
86
  - spec/unit/connection_spec.rb
80
87
  - spec/unit/helper_spec.rb
81
88
  - spec/unit/pool_spec.rb
82
89
  - spec/unit/queue_spec.rb
90
+ - spec/unit/states_spec.rb
91
+ - spec/unit/stats_spec.rb
83
92
  - spec/unit/work_spec.rb
84
93
  - spec/unit/worker_spec.rb
85
94
  - tasks/benchmark.rb
@@ -105,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
105
114
  version: '0'
106
115
  requirements: []
107
116
  rubyforge_project:
108
- rubygems_version: 2.1.5
117
+ rubygems_version: 2.2.0.rc.1
109
118
  signing_key:
110
119
  specification_version: 4
111
120
  summary: A PostgreSQL-based Job Queue
@@ -118,10 +127,12 @@ test_files:
118
127
  - spec/support/helpers.rb
119
128
  - spec/support/jobs.rb
120
129
  - spec/support/shared_examples/adapter.rb
121
- - spec/support/shared_examples/multithreaded_adapter.rb
130
+ - spec/support/shared_examples/multi_threaded_adapter.rb
122
131
  - spec/unit/connection_spec.rb
123
132
  - spec/unit/helper_spec.rb
124
133
  - spec/unit/pool_spec.rb
125
134
  - spec/unit/queue_spec.rb
135
+ - spec/unit/states_spec.rb
136
+ - spec/unit/stats_spec.rb
126
137
  - spec/unit/work_spec.rb
127
138
  - spec/unit/worker_spec.rb