que 0.14.3 → 1.0.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +2 -0
- data/CHANGELOG.md +108 -14
- data/LICENSE.txt +1 -1
- data/README.md +49 -45
- data/bin/command_line_interface.rb +239 -0
- data/bin/que +8 -82
- data/docs/README.md +2 -0
- data/docs/active_job.md +6 -0
- data/docs/advanced_setup.md +7 -64
- data/docs/command_line_interface.md +45 -0
- data/docs/error_handling.md +65 -18
- data/docs/inspecting_the_queue.md +30 -80
- data/docs/job_helper_methods.md +27 -0
- data/docs/logging.md +3 -22
- data/docs/managing_workers.md +6 -61
- data/docs/middleware.md +15 -0
- data/docs/migrating.md +4 -7
- data/docs/multiple_queues.md +8 -4
- data/docs/shutting_down_safely.md +1 -1
- data/docs/using_plain_connections.md +39 -15
- data/docs/using_sequel.md +5 -3
- data/docs/writing_reliable_jobs.md +15 -24
- data/lib/que.rb +98 -182
- data/lib/que/active_job/extensions.rb +97 -0
- data/lib/que/active_record/connection.rb +51 -0
- data/lib/que/active_record/model.rb +48 -0
- data/lib/que/connection.rb +179 -0
- data/lib/que/connection_pool.rb +78 -0
- data/lib/que/job.rb +107 -156
- data/lib/que/job_cache.rb +240 -0
- data/lib/que/job_methods.rb +168 -0
- data/lib/que/listener.rb +176 -0
- data/lib/que/locker.rb +466 -0
- data/lib/que/metajob.rb +47 -0
- data/lib/que/migrations.rb +24 -17
- data/lib/que/migrations/4/down.sql +48 -0
- data/lib/que/migrations/4/up.sql +265 -0
- data/lib/que/poller.rb +267 -0
- data/lib/que/rails/railtie.rb +14 -0
- data/lib/que/result_queue.rb +35 -0
- data/lib/que/sequel/model.rb +51 -0
- data/lib/que/utils/assertions.rb +62 -0
- data/lib/que/utils/constantization.rb +19 -0
- data/lib/que/utils/error_notification.rb +68 -0
- data/lib/que/utils/freeze.rb +20 -0
- data/lib/que/utils/introspection.rb +50 -0
- data/lib/que/utils/json_serialization.rb +21 -0
- data/lib/que/utils/logging.rb +78 -0
- data/lib/que/utils/middleware.rb +33 -0
- data/lib/que/utils/queue_management.rb +18 -0
- data/lib/que/utils/transactions.rb +34 -0
- data/lib/que/version.rb +1 -1
- data/lib/que/worker.rb +128 -167
- data/que.gemspec +13 -2
- metadata +37 -80
- data/.rspec +0 -2
- data/.travis.yml +0 -64
- data/Gemfile +0 -24
- data/docs/customizing_que.md +0 -200
- data/lib/generators/que/install_generator.rb +0 -24
- data/lib/generators/que/templates/add_que.rb +0 -13
- data/lib/que/adapters/active_record.rb +0 -40
- data/lib/que/adapters/base.rb +0 -133
- data/lib/que/adapters/connection_pool.rb +0 -16
- data/lib/que/adapters/pg.rb +0 -21
- data/lib/que/adapters/pond.rb +0 -16
- data/lib/que/adapters/sequel.rb +0 -20
- data/lib/que/railtie.rb +0 -16
- data/lib/que/rake_tasks.rb +0 -59
- data/lib/que/sql.rb +0 -170
- data/spec/adapters/active_record_spec.rb +0 -175
- data/spec/adapters/connection_pool_spec.rb +0 -22
- data/spec/adapters/pg_spec.rb +0 -41
- data/spec/adapters/pond_spec.rb +0 -22
- data/spec/adapters/sequel_spec.rb +0 -57
- data/spec/gemfiles/Gemfile.current +0 -19
- data/spec/gemfiles/Gemfile.old +0 -19
- data/spec/gemfiles/Gemfile.older +0 -19
- data/spec/gemfiles/Gemfile.oldest +0 -19
- data/spec/spec_helper.rb +0 -129
- data/spec/support/helpers.rb +0 -25
- data/spec/support/jobs.rb +0 -35
- data/spec/support/shared_examples/adapter.rb +0 -42
- data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
- data/spec/unit/configuration_spec.rb +0 -31
- data/spec/unit/connection_spec.rb +0 -14
- data/spec/unit/customization_spec.rb +0 -251
- data/spec/unit/enqueue_spec.rb +0 -245
- data/spec/unit/helper_spec.rb +0 -12
- data/spec/unit/logging_spec.rb +0 -101
- data/spec/unit/migrations_spec.rb +0 -84
- data/spec/unit/pool_spec.rb +0 -365
- data/spec/unit/run_spec.rb +0 -14
- data/spec/unit/states_spec.rb +0 -50
- data/spec/unit/stats_spec.rb +0 -46
- data/spec/unit/transaction_spec.rb +0 -36
- data/spec/unit/work_spec.rb +0 -596
- data/spec/unit/worker_spec.rb +0 -167
- data/tasks/benchmark.rb +0 -3
- data/tasks/rspec.rb +0 -14
- data/tasks/safe_shutdown.rb +0 -67
data/spec/unit/worker_spec.rb
DELETED
@@ -1,167 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'spec_helper'
|
4
|
-
|
5
|
-
describe Que::Worker do
|
6
|
-
it "should work jobs when started until there are none available" do
|
7
|
-
begin
|
8
|
-
Que::Job.enqueue
|
9
|
-
Que::Job.enqueue
|
10
|
-
DB[:que_jobs].count.should be 2
|
11
|
-
|
12
|
-
@worker = Que::Worker.new
|
13
|
-
sleep_until { @worker.sleeping? }
|
14
|
-
DB[:que_jobs].count.should be 0
|
15
|
-
|
16
|
-
$logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_worked job_unavailable)
|
17
|
-
|
18
|
-
json = JSON.load($logger.messages[0])
|
19
|
-
json['job']['job_class'].should == 'Que::Job'
|
20
|
-
ensure
|
21
|
-
if @worker
|
22
|
-
@worker.stop
|
23
|
-
@worker.wait_until_stopped
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
it "should work jobs without a named queue by default" do
|
29
|
-
begin
|
30
|
-
Que::Job.enqueue 1
|
31
|
-
Que::Job.enqueue 2, :queue => 'my_queue'
|
32
|
-
|
33
|
-
@worker = Que::Worker.new
|
34
|
-
sleep_until { @worker.sleeping? }
|
35
|
-
DB[:que_jobs].count.should be 1
|
36
|
-
|
37
|
-
$logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
|
38
|
-
|
39
|
-
json = JSON.load($logger.messages[0])
|
40
|
-
json['job']['queue'].should == ''
|
41
|
-
json['job']['job_class'].should == 'Que::Job'
|
42
|
-
json['job']['args'].should == [1]
|
43
|
-
ensure
|
44
|
-
if @worker
|
45
|
-
@worker.stop
|
46
|
-
@worker.wait_until_stopped
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
it "should accept the name of a single queue to work jobs from" do
|
52
|
-
begin
|
53
|
-
Que::Job.enqueue 1
|
54
|
-
Que::Job.enqueue 2, :queue => 'my_queue'
|
55
|
-
|
56
|
-
@worker = Que::Worker.new(:my_queue)
|
57
|
-
sleep_until { @worker.sleeping? }
|
58
|
-
DB[:que_jobs].count.should be 1
|
59
|
-
|
60
|
-
$logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
|
61
|
-
|
62
|
-
json = JSON.load($logger.messages[0])
|
63
|
-
json['job']['queue'].should == 'my_queue'
|
64
|
-
json['job']['job_class'].should == 'Que::Job'
|
65
|
-
json['job']['args'].should == [2]
|
66
|
-
ensure
|
67
|
-
if @worker
|
68
|
-
@worker.stop
|
69
|
-
@worker.wait_until_stopped
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
it "#wake! should return truthy if the worker was asleep and is woken up, at which point it should work until no jobs are available" do
|
75
|
-
begin
|
76
|
-
@worker = Que::Worker.new
|
77
|
-
sleep_until { @worker.sleeping? }
|
78
|
-
|
79
|
-
Que::Job.enqueue
|
80
|
-
Que::Job.enqueue
|
81
|
-
DB[:que_jobs].count.should be 2
|
82
|
-
|
83
|
-
@worker.wake!.should be true
|
84
|
-
sleep_until { @worker.sleeping? }
|
85
|
-
DB[:que_jobs].count.should be 0
|
86
|
-
ensure
|
87
|
-
if @worker
|
88
|
-
@worker.stop
|
89
|
-
@worker.wait_until_stopped
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
it "#wake! should return falsy if the worker was already working" do
|
95
|
-
begin
|
96
|
-
BlockJob.enqueue
|
97
|
-
@worker = Que::Worker.new
|
98
|
-
|
99
|
-
$q1.pop
|
100
|
-
DB[:que_jobs].count.should be 1
|
101
|
-
@worker.wake!.should be nil
|
102
|
-
$q2.push nil
|
103
|
-
ensure
|
104
|
-
if @worker
|
105
|
-
@worker.stop
|
106
|
-
@worker.wait_until_stopped
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
it "should not be deterred by a job that raises an error" do
|
112
|
-
begin
|
113
|
-
ErrorJob.enqueue :priority => 1
|
114
|
-
Que::Job.enqueue :priority => 5
|
115
|
-
|
116
|
-
@worker = Que::Worker.new
|
117
|
-
|
118
|
-
sleep_until { @worker.sleeping? }
|
119
|
-
|
120
|
-
DB[:que_jobs].count.should be 1
|
121
|
-
job = DB[:que_jobs].first
|
122
|
-
job[:job_class].should == 'ErrorJob'
|
123
|
-
job[:run_at].should be_within(3).of Time.now + 4
|
124
|
-
|
125
|
-
log = JSON.load($logger.messages[0])
|
126
|
-
log['event'].should == 'job_errored'
|
127
|
-
log['error']['class'].should == 'RuntimeError'
|
128
|
-
log['error']['message'].should == "ErrorJob!"
|
129
|
-
log['job']['job_class'].should == 'ErrorJob'
|
130
|
-
ensure
|
131
|
-
if @worker
|
132
|
-
@worker.stop
|
133
|
-
@worker.wait_until_stopped
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
137
|
-
|
138
|
-
it "should receive and respect a notification to stop down when it is working, after its current job completes" do
|
139
|
-
begin
|
140
|
-
BlockJob.enqueue :priority => 1
|
141
|
-
Que::Job.enqueue :priority => 5
|
142
|
-
DB[:que_jobs].count.should be 2
|
143
|
-
|
144
|
-
@worker = Que::Worker.new
|
145
|
-
|
146
|
-
$q1.pop
|
147
|
-
@worker.stop
|
148
|
-
$q2.push nil
|
149
|
-
|
150
|
-
@worker.wait_until_stopped
|
151
|
-
|
152
|
-
DB[:que_jobs].count.should be 1
|
153
|
-
job = DB[:que_jobs].first
|
154
|
-
job[:job_class].should == 'Que::Job'
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
it "should receive and respect a notification to stop when it is currently asleep" do
|
159
|
-
begin
|
160
|
-
@worker = Que::Worker.new
|
161
|
-
sleep_until { @worker.sleeping? }
|
162
|
-
|
163
|
-
@worker.stop
|
164
|
-
@worker.wait_until_stopped
|
165
|
-
end
|
166
|
-
end
|
167
|
-
end
|
data/tasks/benchmark.rb
DELETED
data/tasks/rspec.rb
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'rspec/core/rake_task'
|
4
|
-
|
5
|
-
RSpec::Core::RakeTask.new :default do |spec|
|
6
|
-
spec.pattern = './spec/**/*_spec.rb'
|
7
|
-
end
|
8
|
-
|
9
|
-
# Shortcut to skip the adapter specs, and run only with the basic PG
|
10
|
-
# connection. I use this occasionally to make sure ActiveRecord isn't loaded,
|
11
|
-
# so any accidental Rails-isms are caught.
|
12
|
-
RSpec::Core::RakeTask.new :pg do |spec|
|
13
|
-
spec.pattern = './spec/unit/*_spec.rb'
|
14
|
-
end
|
data/tasks/safe_shutdown.rb
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# This task is used to test Que's behavior when its process is shut down.
|
4
|
-
|
5
|
-
# The situation we're trying to avoid occurs when the process dies while a job
|
6
|
-
# is in the middle of a transaction - ideally, the transaction would be rolled
|
7
|
-
# back and the job could just be reattempted later, but if we're not careful,
|
8
|
-
# the transaction could be committed prematurely. For specifics, see here:
|
9
|
-
|
10
|
-
# http://coderrr.wordpress.com/2011/05/03/beware-of-threadkill-or-your-activerecord-transactions-are-in-danger-of-being-partially-committed/
|
11
|
-
|
12
|
-
# So, this task opens a transaction within a job, makes a write, then prompts
|
13
|
-
# you to kill it with one of a few signals. You can then run it again to make
|
14
|
-
# sure that the write was rolled back (if it wasn't, Que isn't functioning
|
15
|
-
# like it should). This task only explicitly tests Sequel, but the behavior
|
16
|
-
# for ActiveRecord is very similar.
|
17
|
-
|
18
|
-
task :safe_shutdown do
|
19
|
-
require 'sequel'
|
20
|
-
require 'que'
|
21
|
-
|
22
|
-
url = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
|
23
|
-
DB = Sequel.connect(url)
|
24
|
-
|
25
|
-
if DB.table_exists?(:que_jobs)
|
26
|
-
puts "Uh-oh! Previous shutdown wasn't clean!" if DB[:que_jobs].where(:job_id => 0).count > 0
|
27
|
-
DB.drop_table :que_jobs
|
28
|
-
end
|
29
|
-
|
30
|
-
Que.connection = DB
|
31
|
-
Que.create!
|
32
|
-
|
33
|
-
$queue = Queue.new
|
34
|
-
|
35
|
-
class SafeJob < Que::Job
|
36
|
-
def run
|
37
|
-
DB.transaction do
|
38
|
-
DB[:que_jobs].insert(:job_id => 0, :job_class => 'Que::Job')
|
39
|
-
$queue.push nil
|
40
|
-
sleep
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
SafeJob.enqueue
|
46
|
-
Que.mode = :async
|
47
|
-
$queue.pop
|
48
|
-
|
49
|
-
puts "From a different terminal window, run one of the following:"
|
50
|
-
%w(SIGINT SIGTERM SIGKILL).each do |signal|
|
51
|
-
puts "kill -#{signal} #{Process.pid}"
|
52
|
-
end
|
53
|
-
|
54
|
-
stop = false
|
55
|
-
trap('INT'){stop = true}
|
56
|
-
|
57
|
-
at_exit do
|
58
|
-
$stdout.puts "Finishing Que's current jobs before exiting..."
|
59
|
-
Que.mode = :off
|
60
|
-
$stdout.puts "Que's jobs finished, exiting..."
|
61
|
-
end
|
62
|
-
|
63
|
-
loop do
|
64
|
-
sleep 0.01
|
65
|
-
break if stop
|
66
|
-
end
|
67
|
-
end
|