que 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/que.gemspec CHANGED
@@ -19,6 +19,4 @@ Gem::Specification.new do |spec|
19
19
  spec.require_paths = ['lib']
20
20
 
21
21
  spec.add_development_dependency 'bundler', '~> 1.3'
22
-
23
- spec.add_dependency 'multi_json', '~> 1.0'
24
22
  end
@@ -14,17 +14,21 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
14
14
  it_behaves_like "a multi-threaded Que adapter"
15
15
 
16
16
  it "should use the same connection that ActiveRecord does" do
17
- class ActiveRecordJob < Que::Job
18
- def run
19
- $pid1 = Que.execute("SELECT pg_backend_pid()").first['pg_backend_pid'].to_i
20
- $pid2 = ActiveRecord::Base.connection.select_all("select pg_backend_pid()").rows.first.first.to_i
17
+ begin
18
+ class ActiveRecordJob < Que::Job
19
+ def run
20
+ $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
21
+ $pid2 = Integer(ActiveRecord::Base.connection.select_value("select pg_backend_pid()"))
22
+ end
21
23
  end
22
- end
23
24
 
24
- ActiveRecordJob.queue
25
- Que::Job.work
25
+ ActiveRecordJob.queue
26
+ Que::Job.work
26
27
 
27
- $pid1.should == $pid2
28
+ $pid1.should == $pid2
29
+ ensure
30
+ $pid1 = $pid2 = nil
31
+ end
28
32
  end
29
33
 
30
34
  it "should instantiate args as ActiveSupport::HashWithIndifferentAccess" do
@@ -44,5 +48,31 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
44
48
  Que.wake_interval = 0.005.seconds
45
49
  sleep_until { DB[:que_jobs].empty? }
46
50
  end
51
+
52
+ it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
53
+ Que.mode = :async
54
+ sleep_until { Que::Worker.workers.all? &:sleeping? }
55
+
56
+ # Wakes a worker immediately when not in a transaction.
57
+ Que::Job.queue
58
+ sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
59
+
60
+ ActiveRecord::Base.transaction do
61
+ Que::Job.queue
62
+ Que::Worker.workers.each { |worker| worker.should be_sleeping }
63
+ end
64
+ sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
65
+
66
+ # Do nothing when queueing with a specific :run_at.
67
+ BlockJob.queue :run_at => Time.now
68
+ Que::Worker.workers.each { |worker| worker.should be_sleeping }
69
+ end
70
+
71
+ it "should be able to tell when it's in an ActiveRecord transaction" do
72
+ Que.adapter.should_not be_in_transaction
73
+ ActiveRecord::Base.transaction do
74
+ Que.adapter.should be_in_transaction
75
+ end
76
+ end
47
77
  end
48
78
  end
@@ -1,11 +1,20 @@
1
1
  require 'spec_helper'
2
2
  require 'connection_pool'
3
3
 
4
- Que.connection = ConnectionPool.new &NEW_PG_CONNECTION
4
+ Que.connection = QUE_SPEC_CONNECTION_POOL = ConnectionPool.new &NEW_PG_CONNECTION
5
5
  QUE_ADAPTERS[:connection_pool] = Que.adapter
6
6
 
7
7
  describe "Que using the ConnectionPool adapter" do
8
8
  before { Que.adapter = QUE_ADAPTERS[:connection_pool] }
9
9
 
10
10
  it_behaves_like "a multi-threaded Que adapter"
11
+
12
+ it "should be able to tell when it's already in a transaction" do
13
+ Que.adapter.should_not be_in_transaction
14
+ QUE_SPEC_CONNECTION_POOL.with do |conn|
15
+ conn.async_exec "BEGIN"
16
+ Que.adapter.should be_in_transaction
17
+ conn.async_exec "COMMIT"
18
+ end
19
+ end
11
20
  end
@@ -9,17 +9,21 @@ describe "Que using the Sequel adapter" do
9
9
  it_behaves_like "a multi-threaded Que adapter"
10
10
 
11
11
  it "should use the same connection that Sequel does" do
12
- class SequelJob < Que::Job
13
- def run
14
- $pid1 = Que.execute("SELECT pg_backend_pid()").first['pg_backend_pid'].to_i
15
- $pid2 = SEQUEL_ADAPTER_DB.get{pg_backend_pid{}}
12
+ begin
13
+ class SequelJob < Que::Job
14
+ def run
15
+ $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
16
+ $pid2 = Integer(SEQUEL_ADAPTER_DB['select pg_backend_pid()'].get)
17
+ end
16
18
  end
17
- end
18
19
 
19
- SequelJob.queue
20
- Que::Job.work
20
+ SequelJob.queue
21
+ Que::Job.work
21
22
 
22
- $pid1.should == $pid2
23
+ $pid1.should == $pid2
24
+ ensure
25
+ $pid1 = $pid2 = nil
26
+ end
23
27
  end
24
28
 
25
29
  it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
@@ -40,4 +44,11 @@ describe "Que using the Sequel adapter" do
40
44
  BlockJob.queue :run_at => Time.now
41
45
  Que::Worker.workers.each { |worker| worker.should be_sleeping }
42
46
  end
47
+
48
+ it "should be able to tell when it's in a Sequel transaction" do
49
+ Que.adapter.should_not be_in_transaction
50
+ SEQUEL_ADAPTER_DB.transaction do
51
+ Que.adapter.should be_in_transaction
52
+ end
53
+ end
43
54
  end
data/spec/spec_helper.rb CHANGED
@@ -1,8 +1,8 @@
1
1
  require 'que'
2
2
  require 'uri'
3
3
  require 'pg'
4
- require 'json'
5
4
  require 'logger'
5
+ require 'json'
6
6
 
7
7
  Dir['./spec/support/**/*.rb'].sort.each &method(:require)
8
8
 
@@ -42,8 +42,12 @@ QUE_ADAPTERS = {:pg => Que.adapter}
42
42
  # We use Sequel to examine the database in specs.
43
43
  require 'sequel'
44
44
  DB = Sequel.connect(QUE_URL)
45
+
46
+
47
+
48
+ # Reset the table to the most up-to-date version.
45
49
  DB.drop_table? :que_jobs
46
- DB.run Que::SQL[:create_table]
50
+ Que::Migrations.migrate!
47
51
 
48
52
 
49
53
 
@@ -7,7 +7,9 @@ shared_examples "a Que adapter" do
7
7
 
8
8
  it "should be able to queue and work a job" do
9
9
  Que::Job.queue
10
- Que::Job.work.should be_an_instance_of Que::Job
10
+ result = Que::Job.work
11
+ result[:event].should == :job_worked
12
+ result[:job][:job_class].should == 'Que::Job'
11
13
  end
12
14
 
13
15
  it "should yield the same Postgres connection for the duration of the block" do
data/spec/travis.rb ADDED
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Run tests a bunch of times, flush out thread race conditions / errors.
4
+ test_runs = if ENV['TESTS']
5
+ Integer(ENV['TESTS'])
6
+ else
7
+ 50
8
+ end
9
+
10
+ 1.upto(test_runs) do |i|
11
+ puts "Test Run #{i}"
12
+ exit(-1) if !system("bundle exec rake")
13
+ end
@@ -1,14 +1,6 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Que, 'helpers' do
4
- it "should be able to drop and create the jobs table" do
5
- DB.table_exists?(:que_jobs).should be true
6
- Que.drop!
7
- DB.table_exists?(:que_jobs).should be false
8
- Que.create!
9
- DB.table_exists?(:que_jobs).should be true
10
- end
11
-
12
4
  it "should be able to clear the jobs table" do
13
5
  DB[:que_jobs].insert :job_class => "Que::Job"
14
6
  DB[:que_jobs].count.should be 1
@@ -0,0 +1,75 @@
1
+ require 'spec_helper'
2
+
3
+ describe "Logging" do
4
+ it "by default should record the library and thread id in JSON" do
5
+ Que.log :event => "blah", :source => 4
6
+ $logger.messages.count.should be 1
7
+
8
+ message = JSON.load($logger.messages.first)
9
+ message['lib'].should == 'que'
10
+ message['event'].should == 'blah'
11
+ message['source'].should == 4
12
+ message['thread'].should == Thread.current.object_id
13
+ end
14
+
15
+ it "should not raise an error when no logger is present" do
16
+ begin
17
+ Que.logger = nil
18
+
19
+ Que::Job.queue
20
+ worker = Que::Worker.new
21
+ sleep_until { worker.sleeping? }
22
+
23
+ DB[:que_jobs].should be_empty
24
+
25
+ worker.stop
26
+ worker.wait_until_stopped
27
+ ensure
28
+ Que.logger = $logger
29
+ end
30
+ end
31
+
32
+ it "should allow the use of a custom log formatter" do
33
+ begin
34
+ Que.log_formatter = proc { |data| "Logged event is #{data[:event]}" }
35
+ Que.log :event => 'my_event'
36
+ $logger.messages.count.should be 1
37
+ $logger.messages.first.should == "Logged event is my_event"
38
+ ensure
39
+ Que.log_formatter = nil
40
+ end
41
+ end
42
+
43
+ it "should not log anything if the logging formatter returns falsey" do
44
+ begin
45
+ Que.log_formatter = proc { |data| false }
46
+
47
+ Que.log :event => "blah"
48
+ $logger.messages.should be_empty
49
+ ensure
50
+ Que.log_formatter = nil
51
+ end
52
+ end
53
+
54
+ it "should use a :level option to set the log level if one exists, or default to info" do
55
+ begin
56
+ Que.logger = o = Object.new
57
+
58
+ def o.method_missing(level, message)
59
+ $level = level
60
+ $message = message
61
+ end
62
+
63
+ Que.log :message => 'one'
64
+ $level.should == :info
65
+ JSON.load($message)['message'].should == 'one'
66
+
67
+ Que.log :message => 'two', :level => 'debug'
68
+ $level.should == :debug
69
+ JSON.load($message)['message'].should == 'two'
70
+ ensure
71
+ Que.logger = $logger
72
+ $level = $message = nil
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,114 @@
1
+ require 'spec_helper'
2
+
3
+ describe Que::Migrations do
4
+ it "should be able to perform migrations up and down" do
5
+ # Migration #1 creates the table with a priority default of 1, migration
6
+ # #2 ups that to 100.
7
+
8
+ default = proc do
9
+ result = Que.execute <<-SQL
10
+ select adsrc
11
+ from pg_attribute a
12
+ join pg_class c on c.oid = a.attrelid
13
+ join pg_attrdef on adrelid = attrelid AND adnum = attnum
14
+ where relname = 'que_jobs'
15
+ and attname = 'priority'
16
+ SQL
17
+
18
+ result.first['adsrc'].to_i
19
+ end
20
+
21
+ default.call.should == 100
22
+ Que::Migrations.migrate! :version => 1
23
+ default.call.should == 1
24
+ Que::Migrations.migrate! :version => 2
25
+ default.call.should == 100
26
+
27
+ # Clean up.
28
+ Que.migrate!
29
+ end
30
+
31
+ it "should be able to get and set the current schema version" do
32
+ Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
33
+ Que::Migrations.set_db_version(59328)
34
+ Que::Migrations.db_version.should == 59328
35
+ Que::Migrations.set_db_version(Que::Migrations::CURRENT_VERSION)
36
+ Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
37
+ end
38
+
39
+ it "should be able to cycle the jobs table all the way between nonexistent and current without error" do
40
+ Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
41
+ Que::Migrations.migrate! :version => 0
42
+ Que::Migrations.db_version.should == 0
43
+ Que::Migrations.migrate!
44
+ Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
45
+
46
+ # The helper on the Que module does the same thing.
47
+ Que.migrate! :version => 0
48
+ Que::Migrations.db_version.should == 0
49
+ Que.migrate!
50
+ Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
51
+ end
52
+
53
+ it "should be able to honor the initial behavior of Que.drop!" do
54
+ DB.table_exists?(:que_jobs).should be true
55
+ Que.drop!
56
+ DB.table_exists?(:que_jobs).should be false
57
+
58
+ # Clean up.
59
+ Que::Migrations.migrate!
60
+ DB.table_exists?(:que_jobs).should be true
61
+ end
62
+
63
+ it "should be able to recognize a que_jobs table created before the versioning system" do
64
+ DB.drop_table :que_jobs
65
+ DB.create_table(:que_jobs){serial :id} # Dummy Table.
66
+ Que::Migrations.db_version.should == 1
67
+ DB.drop_table(:que_jobs)
68
+ Que::Migrations.migrate!
69
+ end
70
+
71
+ it "should be able to honor the initial behavior of Que.create!" do
72
+ DB.drop_table :que_jobs
73
+ Que.create!
74
+ DB.table_exists?(:que_jobs).should be true
75
+ Que::Migrations.db_version.should == 1
76
+
77
+ # Clean up.
78
+ Que::Migrations.migrate!
79
+ DB.table_exists?(:que_jobs).should be true
80
+ end
81
+
82
+ it "should use transactions to protect its migrations from errors" do
83
+ proc do
84
+ Que::Migrations.transaction do
85
+ Que.execute "DROP TABLE que_jobs"
86
+ Que.execute "invalid SQL syntax"
87
+ end
88
+ end.should raise_error(PG::Error)
89
+
90
+ DB.table_exists?(:que_jobs).should be true
91
+ end
92
+
93
+ # In Ruby 1.9, it's impossible to tell inside an ensure block whether the
94
+ # currently executing thread has been killed.
95
+ unless RUBY_VERSION.start_with?('1.9')
96
+ it "should use transactions to protect its migrations from killed threads" do
97
+ q = Queue.new
98
+
99
+ t = Thread.new do
100
+ Que::Migrations.transaction do
101
+ Que.execute "DROP TABLE que_jobs"
102
+ q.push :go!
103
+ sleep
104
+ end
105
+ end
106
+
107
+ q.pop
108
+ t.kill
109
+ t.join
110
+
111
+ DB.table_exists?(:que_jobs).should be true
112
+ end
113
+ end
114
+ end
@@ -4,12 +4,15 @@ describe "Managing the Worker pool" do
4
4
  it "should log mode changes" do
5
5
  Que.mode = :sync
6
6
  Que.mode = :off
7
- $logger.messages.should == ["[Que] Set mode to :sync", "[Que] Set mode to :off"]
8
- end
9
7
 
10
- it "Que.stop! should do nothing if there are no workers running" do
11
- Que::Worker.workers.should be_empty
12
- Que.stop!
8
+ $logger.messages.count.should be 2
9
+ m1, m2 = $logger.messages.map{|m| JSON.load(m)}
10
+
11
+ m1['event'].should == 'mode_change'
12
+ m1['value'].should == 'sync'
13
+
14
+ m2['event'].should == 'mode_change'
15
+ m2['value'].should == 'off'
13
16
  end
14
17
 
15
18
  describe "Que.mode = :sync" do
@@ -19,10 +22,6 @@ describe "Managing the Worker pool" do
19
22
  ArgsJob.queue(5, :testing => "synchronous").should be_an_instance_of ArgsJob
20
23
  $passed_args.should == [5, {'testing' => "synchronous"}]
21
24
  DB[:que_jobs].count.should be 0
22
-
23
- $logger.messages.length.should be 2
24
- $logger.messages[0].should == "[Que] Set mode to :sync"
25
- $logger.messages[1].should =~ /\A\[Que\] Worked job in/
26
25
  end
27
26
 
28
27
  it "should not affect jobs that are queued with specific run_ats" do
@@ -31,11 +30,6 @@ describe "Managing the Worker pool" do
31
30
  ArgsJob.queue(5, :testing => "synchronous", :run_at => Time.now + 60)
32
31
  DB[:que_jobs].select_map(:job_class).should == ["ArgsJob"]
33
32
  end
34
-
35
- it "then Que.stop! should do nothing" do
36
- Que::Worker.workers.should be_empty
37
- Que.stop!
38
- end
39
33
  end
40
34
 
41
35
  describe "Que.mode = :async" do
@@ -44,7 +38,8 @@ describe "Managing the Worker pool" do
44
38
  Que.worker_count.should be 4
45
39
  sleep_until { Que::Worker.workers.all?(&:sleeping?) }
46
40
 
47
- $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4
41
+ $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
42
+ [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4
48
43
  end
49
44
 
50
45
  it "should be done automatically when setting a worker count" do
@@ -53,7 +48,8 @@ describe "Managing the Worker pool" do
53
48
  Que.worker_count.should == 2
54
49
  sleep_until { Que::Worker.workers.all?(&:sleeping?) }
55
50
 
56
- $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 2"] + ["[Que] No jobs available..."] * 2
51
+ $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
52
+ [['mode_change', 'async'], ['worker_count_change', '2']] + [['job_unavailable', nil]] * 2
57
53
  end
58
54
 
59
55
  it "should not affect the number of workers if a worker_count has already been set" do
@@ -62,7 +58,8 @@ describe "Managing the Worker pool" do
62
58
  Que.worker_count.should be 1
63
59
  sleep_until { Que::Worker.workers.all?(&:sleeping?) }
64
60
 
65
- $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 1", "[Que] No jobs available..."]
61
+ $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
62
+ [['mode_change', 'async'], ['worker_count_change', '1'], ['job_unavailable', nil]]
66
63
  end
67
64
 
68
65
  it "then Que.worker_count = 0 should set the mode to :off" do
@@ -75,7 +72,8 @@ describe "Managing the Worker pool" do
75
72
  Que.worker_count.should == 0
76
73
  Que.mode.should == :off
77
74
 
78
- $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set mode to :off", "[Que] Set worker_count to 0"]
75
+ $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
76
+ [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['mode_change', 'off'], ['worker_count_change', '0']]
79
77
  end
80
78
 
81
79
  it "then Que.worker_count = 2 should gracefully decrease the number of workers" do
@@ -94,7 +92,8 @@ describe "Managing the Worker pool" do
94
92
  worker.thread.status.should == false
95
93
  end
96
94
 
97
- $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set worker_count to 2"]
95
+ $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
96
+ [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['worker_count_change', '2']]
98
97
  end
99
98
 
100
99
  it "then Que.worker_count = 6 should gracefully increase the number of workers" do
@@ -109,7 +108,8 @@ describe "Managing the Worker pool" do
109
108
 
110
109
  workers.should == Que::Worker.workers[0..3]
111
110
 
112
- $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set worker_count to 6"] + ["[Que] No jobs available..."] * 2
111
+ $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
112
+ [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['worker_count_change', '6']] + [['job_unavailable', nil]] * 2
113
113
  end
114
114
 
115
115
  it "then Que.mode = :off should gracefully shut down workers" do
@@ -124,7 +124,8 @@ describe "Managing the Worker pool" do
124
124
  workers.count.should be 4
125
125
  workers.each { |worker| worker.thread.status.should be false }
126
126
 
127
- $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set mode to :off", "[Que] Set worker_count to 0"]
127
+ $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
128
+ [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['mode_change', 'off'], ['worker_count_change', '0']]
128
129
  end
129
130
 
130
131
  it "then Que.wake! should wake up a single worker" do
@@ -181,24 +182,5 @@ describe "Managing the Worker pool" do
181
182
  Que::Job.queue
182
183
  sleep_until { DB[:que_jobs].count == 0 }
183
184
  end
184
-
185
- it "then Que.stop! should interrupt all running jobs" do
186
- begin
187
- # Que.stop! can unpredictably affect connections, which may affect
188
- # other tests, so use a new one.
189
- pg = NEW_PG_CONNECTION.call
190
- Que.connection = pg
191
-
192
- BlockJob.queue
193
- Que.mode = :async
194
- $q1.pop
195
- Que.stop!
196
- ensure
197
- if pg
198
- # Closing the connection can raise a NullPointerException on JRuby.
199
- pg.close rescue nil
200
- end
201
- end
202
- end
203
185
  end
204
186
  end