que 0.14.3 → 1.0.0.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +2 -0
- data/CHANGELOG.md +108 -14
- data/LICENSE.txt +1 -1
- data/README.md +49 -45
- data/bin/command_line_interface.rb +239 -0
- data/bin/que +8 -82
- data/docs/README.md +2 -0
- data/docs/active_job.md +6 -0
- data/docs/advanced_setup.md +7 -64
- data/docs/command_line_interface.md +45 -0
- data/docs/error_handling.md +65 -18
- data/docs/inspecting_the_queue.md +30 -80
- data/docs/job_helper_methods.md +27 -0
- data/docs/logging.md +3 -22
- data/docs/managing_workers.md +6 -61
- data/docs/middleware.md +15 -0
- data/docs/migrating.md +4 -7
- data/docs/multiple_queues.md +8 -4
- data/docs/shutting_down_safely.md +1 -1
- data/docs/using_plain_connections.md +39 -15
- data/docs/using_sequel.md +5 -3
- data/docs/writing_reliable_jobs.md +15 -24
- data/lib/que.rb +98 -182
- data/lib/que/active_job/extensions.rb +97 -0
- data/lib/que/active_record/connection.rb +51 -0
- data/lib/que/active_record/model.rb +48 -0
- data/lib/que/connection.rb +179 -0
- data/lib/que/connection_pool.rb +78 -0
- data/lib/que/job.rb +107 -156
- data/lib/que/job_cache.rb +240 -0
- data/lib/que/job_methods.rb +168 -0
- data/lib/que/listener.rb +176 -0
- data/lib/que/locker.rb +466 -0
- data/lib/que/metajob.rb +47 -0
- data/lib/que/migrations.rb +24 -17
- data/lib/que/migrations/4/down.sql +48 -0
- data/lib/que/migrations/4/up.sql +265 -0
- data/lib/que/poller.rb +267 -0
- data/lib/que/rails/railtie.rb +14 -0
- data/lib/que/result_queue.rb +35 -0
- data/lib/que/sequel/model.rb +51 -0
- data/lib/que/utils/assertions.rb +62 -0
- data/lib/que/utils/constantization.rb +19 -0
- data/lib/que/utils/error_notification.rb +68 -0
- data/lib/que/utils/freeze.rb +20 -0
- data/lib/que/utils/introspection.rb +50 -0
- data/lib/que/utils/json_serialization.rb +21 -0
- data/lib/que/utils/logging.rb +78 -0
- data/lib/que/utils/middleware.rb +33 -0
- data/lib/que/utils/queue_management.rb +18 -0
- data/lib/que/utils/transactions.rb +34 -0
- data/lib/que/version.rb +1 -1
- data/lib/que/worker.rb +128 -167
- data/que.gemspec +13 -2
- metadata +37 -80
- data/.rspec +0 -2
- data/.travis.yml +0 -64
- data/Gemfile +0 -24
- data/docs/customizing_que.md +0 -200
- data/lib/generators/que/install_generator.rb +0 -24
- data/lib/generators/que/templates/add_que.rb +0 -13
- data/lib/que/adapters/active_record.rb +0 -40
- data/lib/que/adapters/base.rb +0 -133
- data/lib/que/adapters/connection_pool.rb +0 -16
- data/lib/que/adapters/pg.rb +0 -21
- data/lib/que/adapters/pond.rb +0 -16
- data/lib/que/adapters/sequel.rb +0 -20
- data/lib/que/railtie.rb +0 -16
- data/lib/que/rake_tasks.rb +0 -59
- data/lib/que/sql.rb +0 -170
- data/spec/adapters/active_record_spec.rb +0 -175
- data/spec/adapters/connection_pool_spec.rb +0 -22
- data/spec/adapters/pg_spec.rb +0 -41
- data/spec/adapters/pond_spec.rb +0 -22
- data/spec/adapters/sequel_spec.rb +0 -57
- data/spec/gemfiles/Gemfile.current +0 -19
- data/spec/gemfiles/Gemfile.old +0 -19
- data/spec/gemfiles/Gemfile.older +0 -19
- data/spec/gemfiles/Gemfile.oldest +0 -19
- data/spec/spec_helper.rb +0 -129
- data/spec/support/helpers.rb +0 -25
- data/spec/support/jobs.rb +0 -35
- data/spec/support/shared_examples/adapter.rb +0 -42
- data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
- data/spec/unit/configuration_spec.rb +0 -31
- data/spec/unit/connection_spec.rb +0 -14
- data/spec/unit/customization_spec.rb +0 -251
- data/spec/unit/enqueue_spec.rb +0 -245
- data/spec/unit/helper_spec.rb +0 -12
- data/spec/unit/logging_spec.rb +0 -101
- data/spec/unit/migrations_spec.rb +0 -84
- data/spec/unit/pool_spec.rb +0 -365
- data/spec/unit/run_spec.rb +0 -14
- data/spec/unit/states_spec.rb +0 -50
- data/spec/unit/stats_spec.rb +0 -46
- data/spec/unit/transaction_spec.rb +0 -36
- data/spec/unit/work_spec.rb +0 -596
- data/spec/unit/worker_spec.rb +0 -167
- data/tasks/benchmark.rb +0 -3
- data/tasks/rspec.rb +0 -14
- data/tasks/safe_shutdown.rb +0 -67
data/spec/unit/worker_spec.rb
DELETED
@@ -1,167 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'spec_helper'
|
4
|
-
|
5
|
-
describe Que::Worker do
|
6
|
-
it "should work jobs when started until there are none available" do
|
7
|
-
begin
|
8
|
-
Que::Job.enqueue
|
9
|
-
Que::Job.enqueue
|
10
|
-
DB[:que_jobs].count.should be 2
|
11
|
-
|
12
|
-
@worker = Que::Worker.new
|
13
|
-
sleep_until { @worker.sleeping? }
|
14
|
-
DB[:que_jobs].count.should be 0
|
15
|
-
|
16
|
-
$logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_worked job_unavailable)
|
17
|
-
|
18
|
-
json = JSON.load($logger.messages[0])
|
19
|
-
json['job']['job_class'].should == 'Que::Job'
|
20
|
-
ensure
|
21
|
-
if @worker
|
22
|
-
@worker.stop
|
23
|
-
@worker.wait_until_stopped
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
it "should work jobs without a named queue by default" do
|
29
|
-
begin
|
30
|
-
Que::Job.enqueue 1
|
31
|
-
Que::Job.enqueue 2, :queue => 'my_queue'
|
32
|
-
|
33
|
-
@worker = Que::Worker.new
|
34
|
-
sleep_until { @worker.sleeping? }
|
35
|
-
DB[:que_jobs].count.should be 1
|
36
|
-
|
37
|
-
$logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
|
38
|
-
|
39
|
-
json = JSON.load($logger.messages[0])
|
40
|
-
json['job']['queue'].should == ''
|
41
|
-
json['job']['job_class'].should == 'Que::Job'
|
42
|
-
json['job']['args'].should == [1]
|
43
|
-
ensure
|
44
|
-
if @worker
|
45
|
-
@worker.stop
|
46
|
-
@worker.wait_until_stopped
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
it "should accept the name of a single queue to work jobs from" do
|
52
|
-
begin
|
53
|
-
Que::Job.enqueue 1
|
54
|
-
Que::Job.enqueue 2, :queue => 'my_queue'
|
55
|
-
|
56
|
-
@worker = Que::Worker.new(:my_queue)
|
57
|
-
sleep_until { @worker.sleeping? }
|
58
|
-
DB[:que_jobs].count.should be 1
|
59
|
-
|
60
|
-
$logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
|
61
|
-
|
62
|
-
json = JSON.load($logger.messages[0])
|
63
|
-
json['job']['queue'].should == 'my_queue'
|
64
|
-
json['job']['job_class'].should == 'Que::Job'
|
65
|
-
json['job']['args'].should == [2]
|
66
|
-
ensure
|
67
|
-
if @worker
|
68
|
-
@worker.stop
|
69
|
-
@worker.wait_until_stopped
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
it "#wake! should return truthy if the worker was asleep and is woken up, at which point it should work until no jobs are available" do
|
75
|
-
begin
|
76
|
-
@worker = Que::Worker.new
|
77
|
-
sleep_until { @worker.sleeping? }
|
78
|
-
|
79
|
-
Que::Job.enqueue
|
80
|
-
Que::Job.enqueue
|
81
|
-
DB[:que_jobs].count.should be 2
|
82
|
-
|
83
|
-
@worker.wake!.should be true
|
84
|
-
sleep_until { @worker.sleeping? }
|
85
|
-
DB[:que_jobs].count.should be 0
|
86
|
-
ensure
|
87
|
-
if @worker
|
88
|
-
@worker.stop
|
89
|
-
@worker.wait_until_stopped
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
it "#wake! should return falsy if the worker was already working" do
|
95
|
-
begin
|
96
|
-
BlockJob.enqueue
|
97
|
-
@worker = Que::Worker.new
|
98
|
-
|
99
|
-
$q1.pop
|
100
|
-
DB[:que_jobs].count.should be 1
|
101
|
-
@worker.wake!.should be nil
|
102
|
-
$q2.push nil
|
103
|
-
ensure
|
104
|
-
if @worker
|
105
|
-
@worker.stop
|
106
|
-
@worker.wait_until_stopped
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
it "should not be deterred by a job that raises an error" do
|
112
|
-
begin
|
113
|
-
ErrorJob.enqueue :priority => 1
|
114
|
-
Que::Job.enqueue :priority => 5
|
115
|
-
|
116
|
-
@worker = Que::Worker.new
|
117
|
-
|
118
|
-
sleep_until { @worker.sleeping? }
|
119
|
-
|
120
|
-
DB[:que_jobs].count.should be 1
|
121
|
-
job = DB[:que_jobs].first
|
122
|
-
job[:job_class].should == 'ErrorJob'
|
123
|
-
job[:run_at].should be_within(3).of Time.now + 4
|
124
|
-
|
125
|
-
log = JSON.load($logger.messages[0])
|
126
|
-
log['event'].should == 'job_errored'
|
127
|
-
log['error']['class'].should == 'RuntimeError'
|
128
|
-
log['error']['message'].should == "ErrorJob!"
|
129
|
-
log['job']['job_class'].should == 'ErrorJob'
|
130
|
-
ensure
|
131
|
-
if @worker
|
132
|
-
@worker.stop
|
133
|
-
@worker.wait_until_stopped
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
137
|
-
|
138
|
-
it "should receive and respect a notification to stop down when it is working, after its current job completes" do
|
139
|
-
begin
|
140
|
-
BlockJob.enqueue :priority => 1
|
141
|
-
Que::Job.enqueue :priority => 5
|
142
|
-
DB[:que_jobs].count.should be 2
|
143
|
-
|
144
|
-
@worker = Que::Worker.new
|
145
|
-
|
146
|
-
$q1.pop
|
147
|
-
@worker.stop
|
148
|
-
$q2.push nil
|
149
|
-
|
150
|
-
@worker.wait_until_stopped
|
151
|
-
|
152
|
-
DB[:que_jobs].count.should be 1
|
153
|
-
job = DB[:que_jobs].first
|
154
|
-
job[:job_class].should == 'Que::Job'
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
it "should receive and respect a notification to stop when it is currently asleep" do
|
159
|
-
begin
|
160
|
-
@worker = Que::Worker.new
|
161
|
-
sleep_until { @worker.sleeping? }
|
162
|
-
|
163
|
-
@worker.stop
|
164
|
-
@worker.wait_until_stopped
|
165
|
-
end
|
166
|
-
end
|
167
|
-
end
|
data/tasks/benchmark.rb
DELETED
data/tasks/rspec.rb
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'rspec/core/rake_task'
|
4
|
-
|
5
|
-
RSpec::Core::RakeTask.new :default do |spec|
|
6
|
-
spec.pattern = './spec/**/*_spec.rb'
|
7
|
-
end
|
8
|
-
|
9
|
-
# Shortcut to skip the adapter specs, and run only with the basic PG
|
10
|
-
# connection. I use this occasionally to make sure ActiveRecord isn't loaded,
|
11
|
-
# so any accidental Rails-isms are caught.
|
12
|
-
RSpec::Core::RakeTask.new :pg do |spec|
|
13
|
-
spec.pattern = './spec/unit/*_spec.rb'
|
14
|
-
end
|
data/tasks/safe_shutdown.rb
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This task is used to test Que's behavior when its process is shut down.
|
4
|
-
|
5
|
-
# The situation we're trying to avoid occurs when the process dies while a job
|
6
|
-
# is in the middle of a transaction - ideally, the transaction would be rolled
|
7
|
-
# back and the job could just be reattempted later, but if we're not careful,
|
8
|
-
# the transaction could be committed prematurely. For specifics, see here:
|
9
|
-
|
10
|
-
# http://coderrr.wordpress.com/2011/05/03/beware-of-threadkill-or-your-activerecord-transactions-are-in-danger-of-being-partially-committed/
|
11
|
-
|
12
|
-
# So, this task opens a transaction within a job, makes a write, then prompts
|
13
|
-
# you to kill it with one of a few signals. You can then run it again to make
|
14
|
-
# sure that the write was rolled back (if it wasn't, Que isn't functioning
|
15
|
-
# like it should). This task only explicitly tests Sequel, but the behavior
|
16
|
-
# for ActiveRecord is very similar.
|
17
|
-
|
18
|
-
task :safe_shutdown do
|
19
|
-
require 'sequel'
|
20
|
-
require 'que'
|
21
|
-
|
22
|
-
url = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
|
23
|
-
DB = Sequel.connect(url)
|
24
|
-
|
25
|
-
if DB.table_exists?(:que_jobs)
|
26
|
-
puts "Uh-oh! Previous shutdown wasn't clean!" if DB[:que_jobs].where(:job_id => 0).count > 0
|
27
|
-
DB.drop_table :que_jobs
|
28
|
-
end
|
29
|
-
|
30
|
-
Que.connection = DB
|
31
|
-
Que.create!
|
32
|
-
|
33
|
-
$queue = Queue.new
|
34
|
-
|
35
|
-
class SafeJob < Que::Job
|
36
|
-
def run
|
37
|
-
DB.transaction do
|
38
|
-
DB[:que_jobs].insert(:job_id => 0, :job_class => 'Que::Job')
|
39
|
-
$queue.push nil
|
40
|
-
sleep
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
SafeJob.enqueue
|
46
|
-
Que.mode = :async
|
47
|
-
$queue.pop
|
48
|
-
|
49
|
-
puts "From a different terminal window, run one of the following:"
|
50
|
-
%w(SIGINT SIGTERM SIGKILL).each do |signal|
|
51
|
-
puts "kill -#{signal} #{Process.pid}"
|
52
|
-
end
|
53
|
-
|
54
|
-
stop = false
|
55
|
-
trap('INT'){stop = true}
|
56
|
-
|
57
|
-
at_exit do
|
58
|
-
$stdout.puts "Finishing Que's current jobs before exiting..."
|
59
|
-
Que.mode = :off
|
60
|
-
$stdout.puts "Que's jobs finished, exiting..."
|
61
|
-
end
|
62
|
-
|
63
|
-
loop do
|
64
|
-
sleep 0.01
|
65
|
-
break if stop
|
66
|
-
end
|
67
|
-
end
|