que 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -1
  3. data/.travis.yml +1 -1
  4. data/CHANGELOG.md +21 -1
  5. data/Gemfile +5 -0
  6. data/README.md +7 -6
  7. data/docs/advanced_setup.md +14 -4
  8. data/docs/customizing_que.md +4 -4
  9. data/docs/error_handling.md +13 -1
  10. data/docs/managing_workers.md +2 -2
  11. data/docs/migrating.md +26 -0
  12. data/docs/multiple_queues.md +13 -0
  13. data/docs/shutting_down_safely.md +7 -0
  14. data/docs/writing_reliable_jobs.md +43 -0
  15. data/lib/generators/que/templates/add_que.rb +1 -1
  16. data/lib/que.rb +27 -41
  17. data/lib/que/adapters/base.rb +75 -4
  18. data/lib/que/job.rb +45 -28
  19. data/lib/que/migrations.rb +3 -2
  20. data/lib/que/migrations/{1-down.sql → 1/down.sql} +0 -0
  21. data/lib/que/migrations/{1-up.sql → 1/up.sql} +0 -0
  22. data/lib/que/migrations/{2-down.sql → 2/down.sql} +0 -0
  23. data/lib/que/migrations/{2-up.sql → 2/up.sql} +0 -0
  24. data/lib/que/migrations/3/down.sql +5 -0
  25. data/lib/que/migrations/3/up.sql +5 -0
  26. data/lib/que/sql.rb +24 -17
  27. data/lib/que/version.rb +1 -1
  28. data/lib/que/worker.rb +6 -5
  29. data/spec/adapters/active_record_spec.rb +6 -6
  30. data/spec/adapters/sequel_spec.rb +4 -4
  31. data/spec/gemfiles/Gemfile1 +18 -0
  32. data/spec/gemfiles/Gemfile2 +18 -0
  33. data/spec/support/helpers.rb +2 -1
  34. data/spec/support/shared_examples/adapter.rb +7 -3
  35. data/spec/support/shared_examples/multi_threaded_adapter.rb +2 -2
  36. data/spec/travis.rb +12 -4
  37. data/spec/unit/customization_spec.rb +148 -0
  38. data/spec/unit/{queue_spec.rb → enqueue_spec.rb} +115 -14
  39. data/spec/unit/logging_spec.rb +3 -2
  40. data/spec/unit/migrations_spec.rb +3 -2
  41. data/spec/unit/pool_spec.rb +30 -6
  42. data/spec/unit/run_spec.rb +12 -0
  43. data/spec/unit/states_spec.rb +29 -31
  44. data/spec/unit/stats_spec.rb +16 -14
  45. data/spec/unit/work_spec.rb +120 -25
  46. data/spec/unit/worker_spec.rb +55 -9
  47. data/tasks/safe_shutdown.rb +1 -1
  48. metadata +30 -17
@@ -17,7 +17,7 @@ describe "Que using the Sequel adapter" do
17
17
  end
18
18
  end
19
19
 
20
- SequelJob.queue
20
+ SequelJob.enqueue
21
21
  Que::Job.work
22
22
 
23
23
  $pid1.should == $pid2
@@ -31,17 +31,17 @@ describe "Que using the Sequel adapter" do
31
31
  sleep_until { Que::Worker.workers.all? &:sleeping? }
32
32
 
33
33
  # Wakes a worker immediately when not in a transaction.
34
- Que::Job.queue
34
+ Que::Job.enqueue
35
35
  sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
36
36
 
37
37
  SEQUEL_ADAPTER_DB.transaction do
38
- Que::Job.queue
38
+ Que::Job.enqueue
39
39
  Que::Worker.workers.each { |worker| worker.should be_sleeping }
40
40
  end
41
41
  sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
42
42
 
43
43
  # Do nothing when queueing with a specific :run_at.
44
- BlockJob.queue :run_at => Time.now
44
+ BlockJob.enqueue :run_at => Time.now
45
45
  Que::Worker.workers.each { |worker| worker.should be_sleeping }
46
46
  end
47
47
 
@@ -0,0 +1,18 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'que', path: '../..'
4
+
5
+ group :development, :test do
6
+ gem 'rake'
7
+
8
+ gem 'activerecord', '~> 3.2', :require => nil
9
+ gem 'sequel', '~> 3', :require => nil
10
+ gem 'connection_pool', :require => nil
11
+ gem 'pg', :require => nil, :platform => :ruby
12
+ gem 'pg_jruby', :require => nil, :platform => :jruby
13
+ end
14
+
15
+ group :test do
16
+ gem 'rspec', '~> 2.14.1'
17
+ gem 'pry'
18
+ end
@@ -0,0 +1,18 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'que', path: "../.."
4
+
5
+ group :development, :test do
6
+ gem 'rake'
7
+
8
+ gem 'activerecord', '~> 4.0', :require => nil
9
+ gem 'sequel', '~> 4', :require => nil
10
+ gem 'connection_pool', :require => nil
11
+ gem 'pg', :require => nil, :platform => :ruby
12
+ gem 'pg_jruby', :require => nil, :platform => :jruby
13
+ end
14
+
15
+ group :test do
16
+ gem 'rspec', '~> 2.14.1'
17
+ gem 'pry'
18
+ end
@@ -1,5 +1,6 @@
1
1
  # Helper for testing threaded code.
2
- def sleep_until(timeout = 2)
2
+ QUE_TEST_TIMEOUT ||= 2
3
+ def sleep_until(timeout = QUE_TEST_TIMEOUT)
3
4
  deadline = Time.now + timeout
4
5
  loop do
5
6
  break if yield
@@ -1,12 +1,16 @@
1
1
  shared_examples "a Que adapter" do
2
2
  it "should be able to execute arbitrary SQL and return indifferent hashes" do
3
3
  result = Que.execute("SELECT 1 AS one")
4
- result.should == [{'one'=>'1'}]
5
- result.first[:one].should == '1'
4
+ result.should == [{'one'=>1}]
5
+ result.first[:one].should == 1
6
+ end
7
+
8
+ it "should be able to execute multiple SQL statements in one string" do
9
+ Que.execute("SELECT 1 AS one; SELECT 1 AS one")
6
10
  end
7
11
 
8
12
  it "should be able to queue and work a job" do
9
- Que::Job.queue
13
+ Que::Job.enqueue
10
14
  result = Que::Job.work
11
15
  result[:event].should == :job_worked
12
16
  result[:job][:job_class].should == 'Que::Job'
@@ -26,11 +26,11 @@ shared_examples "a multi-threaded Que adapter" do
26
26
  end
27
27
 
28
28
  it "should allow multiple workers to complete jobs simultaneously" do
29
- BlockJob.queue
29
+ BlockJob.enqueue
30
30
  worker_1 = Que::Worker.new
31
31
  $q1.pop
32
32
 
33
- Que::Job.queue
33
+ Que::Job.enqueue
34
34
  DB[:que_jobs].count.should be 2
35
35
 
36
36
  worker_2 = Que::Worker.new
data/spec/travis.rb CHANGED
@@ -4,10 +4,18 @@
4
4
  test_runs = if ENV['TESTS']
5
5
  Integer(ENV['TESTS'])
6
6
  else
7
- 50
7
+ 25
8
8
  end
9
9
 
10
- 1.upto(test_runs) do |i|
11
- puts "Test Run #{i}"
12
- exit(-1) if !system("bundle exec rake")
10
+
11
+ # I think travis might be pausing jobs, let's try a higher timeout
12
+ QUE_TEST_TIMEOUT = 10
13
+
14
+ %w( Gemfile spec/gemfiles/Gemfile1 spec/gemfiles/Gemfile2 ).each do |gemfile|
15
+ # Install the particular gemfile
16
+ system("BUNDLE_GEMFILE=#{gemfile} bundle")
17
+ 1.upto(test_runs) do |i|
18
+ puts "Test Run #{i}"
19
+ exit(-1) if !system("bundle exec rake")
20
+ end
13
21
  end
@@ -0,0 +1,148 @@
1
+ require 'spec_helper'
2
+
3
+ # A few specs to ensure that the ideas given in the customizing_que document
4
+ # stay functional.
5
+ describe "Customizing Que" do
6
+ it "Cron should allow for easy recurring jobs" do
7
+ class Cron < Que::Job
8
+ def run
9
+ destroy
10
+ self.class.enqueue :run_at => @attrs[:run_at] + 3600
11
+ end
12
+ end
13
+
14
+ Cron.enqueue
15
+
16
+ run_at = DB[:que_jobs].get(:run_at).to_f
17
+
18
+ Que::Job.work
19
+
20
+ DB[:que_jobs].get(:run_at).to_f.should be_within(0.000001).of(run_at + 3600)
21
+ end
22
+
23
+ it "Object#delay should allow for simpler job enqueueing" do
24
+ begin
25
+ class Delayed < Que::Job
26
+ def run(receiver, method, args)
27
+ Marshal.load(receiver).send method, *Marshal.load(args)
28
+ end
29
+ end
30
+
31
+ class DelayedAction
32
+ def initialize(receiver)
33
+ @receiver = receiver
34
+ end
35
+
36
+ def method_missing(method, *args)
37
+ Delayed.queue Marshal.dump(@receiver), method, Marshal.dump(args)
38
+ end
39
+ end
40
+
41
+ class Object
42
+ def delay
43
+ DelayedAction.new(self)
44
+ end
45
+ end
46
+
47
+ module MyModule
48
+ class << self
49
+ def blah
50
+ $run = true
51
+ end
52
+ end
53
+ end
54
+
55
+ MyModule.delay.blah
56
+ Que::Job.work
57
+
58
+ $run.should be true
59
+ ensure
60
+ $run = nil
61
+ end
62
+ end
63
+
64
+ it "QueueClassic-style jobs should be easy" do
65
+ begin
66
+ class Command < Que::Job
67
+ def run(method, *args)
68
+ receiver, message = method.split('.')
69
+ Object.const_get(receiver).send(message, *args)
70
+ end
71
+ end
72
+
73
+ module MyModule
74
+ class << self
75
+ def blah(arg)
76
+ $value = arg
77
+ end
78
+ end
79
+ end
80
+
81
+ Command.enqueue "MyModule.blah", "hello world"
82
+ Que::Job.work
83
+
84
+ $value.should == "hello world"
85
+ ensure
86
+ $value = nil
87
+ end
88
+ end
89
+
90
+ describe "retaining deleted jobs" do
91
+ before do
92
+ Que.execute "CREATE TABLE finished_jobs AS SELECT * FROM que_jobs LIMIT 0"
93
+ end
94
+
95
+ after do
96
+ DB.drop_table? :finished_jobs
97
+ end
98
+
99
+ it "with a Ruby override" do
100
+ class MyJobClass < Que::Job
101
+ def destroy
102
+ Que.execute "INSERT INTO finished_jobs SELECT * FROM que_jobs WHERE queue = $1::text AND priority = $2::integer AND run_at = $3::timestamptz AND job_id = $4::bigint", @attrs.values_at(:queue, :priority, :run_at, :job_id)
103
+ super
104
+ end
105
+ end
106
+
107
+ class MyJob < MyJobClass
108
+ end
109
+
110
+ MyJob.enqueue 1, 'arg1', :priority => 89
111
+ Que::Job.work
112
+
113
+ DB[:finished_jobs].count.should == 1
114
+ job = DB[:finished_jobs].first
115
+ job[:priority].should == 89
116
+ JSON.load(job[:args]).should == [1, 'arg1']
117
+ end
118
+
119
+ it "with a trigger" do
120
+ begin
121
+ Que.execute <<-SQL
122
+ CREATE FUNCTION please_save_my_job()
123
+ RETURNS trigger
124
+ LANGUAGE plpgsql
125
+ AS $$
126
+ BEGIN
127
+ INSERT INTO finished_jobs SELECT (OLD).*;
128
+ RETURN OLD;
129
+ END;
130
+ $$;
131
+ SQL
132
+
133
+ Que.execute "CREATE TRIGGER keep_all_my_old_jobs BEFORE DELETE ON que_jobs FOR EACH ROW EXECUTE PROCEDURE please_save_my_job();"
134
+
135
+ Que::Job.enqueue 2, 'arg2', :priority => 45
136
+ Que::Job.work
137
+
138
+ DB[:finished_jobs].count.should == 1
139
+ job = DB[:finished_jobs].first
140
+ job[:priority].should == 45
141
+ JSON.load(job[:args]).should == [2, 'arg2']
142
+ ensure
143
+ DB.drop_trigger :que_jobs, :keep_all_my_old_jobs, :if_exists => true
144
+ DB.drop_function :please_save_my_job, :if_exists => true
145
+ end
146
+ end
147
+ end
148
+ end
@@ -1,28 +1,37 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Que::Job, '.queue' do
3
+ describe Que::Job, '.enqueue' do
4
4
  it "should be able to queue a job" do
5
5
  DB[:que_jobs].count.should be 0
6
- result = Que::Job.queue
6
+ result = Que::Job.enqueue
7
7
  DB[:que_jobs].count.should be 1
8
8
 
9
9
  result.should be_an_instance_of Que::Job
10
- result.attrs[:priority].should == '100'
10
+ result.attrs[:queue].should == ''
11
+ result.attrs[:priority].should == 100
11
12
  result.attrs[:args].should == []
12
13
 
13
14
  job = DB[:que_jobs].first
15
+ job[:queue].should == ''
14
16
  job[:priority].should be 100
15
17
  job[:run_at].should be_within(3).of Time.now
16
18
  job[:job_class].should == "Que::Job"
17
19
  JSON.load(job[:args]).should == []
18
20
  end
19
21
 
22
+ it "should be aliased to .queue" do
23
+ DB[:que_jobs].count.should be 0
24
+ Que::Job.queue
25
+ DB[:que_jobs].count.should be 1
26
+ end
27
+
20
28
  it "should be able to queue a job with arguments" do
21
29
  DB[:que_jobs].count.should be 0
22
- Que::Job.queue 1, 'two'
30
+ Que::Job.enqueue 1, 'two'
23
31
  DB[:que_jobs].count.should be 1
24
32
 
25
33
  job = DB[:que_jobs].first
34
+ job[:queue].should == ''
26
35
  job[:priority].should be 100
27
36
  job[:run_at].should be_within(3).of Time.now
28
37
  job[:job_class].should == "Que::Job"
@@ -31,7 +40,7 @@ describe Que::Job, '.queue' do
31
40
 
32
41
  it "should be able to queue a job with complex arguments" do
33
42
  DB[:que_jobs].count.should be 0
34
- Que::Job.queue 1, 'two', :string => "string",
43
+ Que::Job.enqueue 1, 'two', :string => "string",
35
44
  :integer => 5,
36
45
  :array => [1, "two", {:three => 3}],
37
46
  :hash => {:one => 1, :two => 'two', :three => [3]}
@@ -39,6 +48,7 @@ describe Que::Job, '.queue' do
39
48
  DB[:que_jobs].count.should be 1
40
49
 
41
50
  job = DB[:que_jobs].first
51
+ job[:queue].should == ''
42
52
  job[:priority].should be 100
43
53
  job[:run_at].should be_within(3).of Time.now
44
54
  job[:job_class].should == "Que::Job"
@@ -56,10 +66,11 @@ describe Que::Job, '.queue' do
56
66
 
57
67
  it "should be able to queue a job with a specific time to run" do
58
68
  DB[:que_jobs].count.should be 0
59
- Que::Job.queue 1, :run_at => Time.now + 60
69
+ Que::Job.enqueue 1, :run_at => Time.now + 60
60
70
  DB[:que_jobs].count.should be 1
61
71
 
62
72
  job = DB[:que_jobs].first
73
+ job[:queue].should == ''
63
74
  job[:priority].should be 100
64
75
  job[:run_at].should be_within(3).of Time.now + 60
65
76
  job[:job_class].should == "Que::Job"
@@ -68,10 +79,11 @@ describe Que::Job, '.queue' do
68
79
 
69
80
  it "should be able to queue a job with a specific priority" do
70
81
  DB[:que_jobs].count.should be 0
71
- Que::Job.queue 1, :priority => 4
82
+ Que::Job.enqueue 1, :priority => 4
72
83
  DB[:que_jobs].count.should be 1
73
84
 
74
85
  job = DB[:que_jobs].first
86
+ job[:queue].should == ''
75
87
  job[:priority].should be 4
76
88
  job[:run_at].should be_within(3).of Time.now
77
89
  job[:job_class].should == "Que::Job"
@@ -80,59 +92,148 @@ describe Que::Job, '.queue' do
80
92
 
81
93
  it "should be able to queue a job with queueing options in addition to argument options" do
82
94
  DB[:que_jobs].count.should be 0
83
- Que::Job.queue 1, :string => "string", :run_at => Time.now + 60, :priority => 4
95
+ Que::Job.enqueue 1, :string => "string", :run_at => Time.now + 60, :priority => 4
84
96
  DB[:que_jobs].count.should be 1
85
97
 
86
98
  job = DB[:que_jobs].first
99
+ job[:queue].should == ''
87
100
  job[:priority].should be 4
88
101
  job[:run_at].should be_within(3).of Time.now + 60
89
102
  job[:job_class].should == "Que::Job"
90
103
  JSON.load(job[:args]).should == [1, {'string' => 'string'}]
91
104
  end
92
105
 
106
+ it "should respect a job class defined as a string" do
107
+ Que.enqueue 'argument', :queue => 'my_queue', :other_arg => 'other_arg', :job_class => 'MyJobClass'
108
+ Que::Job.enqueue 'argument', :queue => 'my_queue', :other_arg => 'other_arg', :job_class => 'MyJobClass'
109
+
110
+ DB[:que_jobs].count.should be 2
111
+ DB[:que_jobs].all.each do |job|
112
+ job[:job_class].should == 'MyJobClass'
113
+ job[:queue].should == 'my_queue'
114
+ JSON.load(job[:args]).should == ['argument', {'other_arg' => 'other_arg'}]
115
+ end
116
+ end
117
+
93
118
  it "should respect a default (but overridable) priority for the job class" do
94
119
  class DefaultPriorityJob < Que::Job
95
- @default_priority = 3
120
+ @priority = 3
96
121
  end
97
122
 
98
123
  DB[:que_jobs].count.should be 0
99
- DefaultPriorityJob.queue 1
100
- DefaultPriorityJob.queue 1, :priority => 4
124
+ DefaultPriorityJob.enqueue 1
125
+ DefaultPriorityJob.enqueue 1, :priority => 4
101
126
  DB[:que_jobs].count.should be 2
102
127
 
103
128
  first, second = DB[:que_jobs].order(:job_id).all
104
129
 
130
+ first[:queue].should == ''
105
131
  first[:priority].should be 3
106
132
  first[:run_at].should be_within(3).of Time.now
107
133
  first[:job_class].should == "DefaultPriorityJob"
108
134
  JSON.load(first[:args]).should == [1]
109
135
 
136
+ second[:queue].should == ''
110
137
  second[:priority].should be 4
111
138
  second[:run_at].should be_within(3).of Time.now
112
139
  second[:job_class].should == "DefaultPriorityJob"
113
140
  JSON.load(second[:args]).should == [1]
114
141
  end
115
142
 
143
+ it "should respect the old @default_priority setting" do
144
+ class OldDefaultPriorityJob < Que::Job
145
+ @default_priority = 3
146
+ end
147
+
148
+ DB[:que_jobs].count.should be 0
149
+ OldDefaultPriorityJob.enqueue 1
150
+ OldDefaultPriorityJob.enqueue 1, :priority => 4
151
+ DB[:que_jobs].count.should be 2
152
+
153
+ first, second = DB[:que_jobs].order(:job_id).all
154
+
155
+ first[:queue].should == ''
156
+ first[:priority].should be 3
157
+ first[:run_at].should be_within(3).of Time.now
158
+ first[:job_class].should == "OldDefaultPriorityJob"
159
+ JSON.load(first[:args]).should == [1]
160
+
161
+ second[:queue].should == ''
162
+ second[:priority].should be 4
163
+ second[:run_at].should be_within(3).of Time.now
164
+ second[:job_class].should == "OldDefaultPriorityJob"
165
+ JSON.load(second[:args]).should == [1]
166
+ end
167
+
116
168
  it "should respect a default (but overridable) run_at for the job class" do
117
169
  class DefaultRunAtJob < Que::Job
118
- @default_run_at = -> { Time.now + 60 }
170
+ @run_at = -> { Time.now + 60 }
119
171
  end
120
172
 
121
173
  DB[:que_jobs].count.should be 0
122
- DefaultRunAtJob.queue 1
123
- DefaultRunAtJob.queue 1, :run_at => Time.now + 30
174
+ DefaultRunAtJob.enqueue 1
175
+ DefaultRunAtJob.enqueue 1, :run_at => Time.now + 30
124
176
  DB[:que_jobs].count.should be 2
125
177
 
126
178
  first, second = DB[:que_jobs].order(:job_id).all
127
179
 
180
+ first[:queue].should == ''
128
181
  first[:priority].should be 100
129
182
  first[:run_at].should be_within(3).of Time.now + 60
130
183
  first[:job_class].should == "DefaultRunAtJob"
131
184
  JSON.load(first[:args]).should == [1]
132
185
 
186
+ second[:queue].should == ''
133
187
  second[:priority].should be 100
134
188
  second[:run_at].should be_within(3).of Time.now + 30
135
189
  second[:job_class].should == "DefaultRunAtJob"
136
190
  JSON.load(second[:args]).should == [1]
137
191
  end
192
+
193
+ it "should respect the old @default_run_at setting" do
194
+ class OldDefaultRunAtJob < Que::Job
195
+ @default_run_at = -> { Time.now + 60 }
196
+ end
197
+
198
+ DB[:que_jobs].count.should be 0
199
+ OldDefaultRunAtJob.enqueue 1
200
+ OldDefaultRunAtJob.enqueue 1, :run_at => Time.now + 30
201
+ DB[:que_jobs].count.should be 2
202
+
203
+ first, second = DB[:que_jobs].order(:job_id).all
204
+
205
+ first[:queue].should == ''
206
+ first[:priority].should be 100
207
+ first[:run_at].should be_within(3).of Time.now + 60
208
+ first[:job_class].should == "OldDefaultRunAtJob"
209
+ JSON.load(first[:args]).should == [1]
210
+
211
+ second[:queue].should == ''
212
+ second[:priority].should be 100
213
+ second[:run_at].should be_within(3).of Time.now + 30
214
+ second[:job_class].should == "OldDefaultRunAtJob"
215
+ JSON.load(second[:args]).should == [1]
216
+ end
217
+
218
+ it "should respect a default (but overridable) queue for the job class" do
219
+ class NamedQueueJob < Que::Job
220
+ @queue = :my_queue
221
+ end
222
+
223
+ DB[:que_jobs].count.should be 0
224
+ NamedQueueJob.enqueue 1
225
+ NamedQueueJob.enqueue 1, :queue => 'my_queue_2'
226
+ NamedQueueJob.enqueue 1, :queue => :my_queue_2
227
+ NamedQueueJob.enqueue 1, :queue => ''
228
+ NamedQueueJob.enqueue 1, :queue => nil
229
+ DB[:que_jobs].count.should be 5
230
+
231
+ first, second, third, fourth, fifth = DB[:que_jobs].order(:job_id).select_map(:queue)
232
+
233
+ first.should == 'my_queue'
234
+ second.should == 'my_queue_2'
235
+ third.should == 'my_queue_2'
236
+ fourth.should == ''
237
+ fifth.should == ''
238
+ end
138
239
  end