que 0.6.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/Gemfile +1 -0
- data/docs/managing_workers.md +4 -0
- data/docs/multiple_queues.md +8 -0
- data/docs/using_plain_connections.md +17 -1
- data/lib/que.rb +7 -5
- data/lib/que/adapters/base.rb +1 -0
- data/lib/que/adapters/pond.rb +14 -0
- data/lib/que/job.rb +9 -2
- data/lib/que/rake_tasks.rb +6 -4
- data/lib/que/version.rb +1 -1
- data/lib/que/worker.rb +1 -0
- data/spec/adapters/pond_spec.rb +20 -0
- data/spec/support/helpers.rb +7 -0
- data/spec/unit/customization_spec.rb +1 -1
- data/spec/unit/enqueue_spec.rb +9 -5
- data/spec/unit/logging_spec.rb +1 -0
- data/spec/unit/work_spec.rb +18 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a18d2e13eaf6870dbd5e912bf072b54d4f45e79d
|
4
|
+
data.tar.gz: 74a6bef98e5c1911dcf944c05392941445f71da5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 22d13ee07b45cab14849d810ffa714d9f773b13a3509f9e5c20e10b1e192c760c4903eed5d8d1721614a29d85b4542d51c3aa440d491047b5cb540e0c497122b
|
7
|
+
data.tar.gz: 8c8cab41047a14bf177e8394439213f4aef0e4688e53f9d99d765ce12d0bd5489db8bd36f335e2e76bf72a19fba4e0f6f3c22670650d3786ff8a6f939ba849dc
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
### 0.7.0 (2014-04-09)
|
2
|
+
|
3
|
+
* `JobClass.queue(*args)` has been deprecated and will be removed in version 1.0.0. Please use `JobClass.enqueue(*args)` instead.
|
4
|
+
|
5
|
+
* The `@default_priority` and `@default_run_at` variables have been deprecated and will be removed in version 1.0.0. Please use `@priority` and `@run_at` instead, respectively.
|
6
|
+
|
7
|
+
* Log lines now include the process pid - its omission in the previous release was an oversight.
|
8
|
+
|
9
|
+
* The [Pond gem](https://github.com/chanks/pond) is now supported as a connection. It is very similar to the ConnectionPool gem, but creates connections lazily and is dynamically resizable.
|
10
|
+
|
1
11
|
### 0.6.0 (2014-02-04)
|
2
12
|
|
3
13
|
* **A schema upgrade to version 3 is required for this release.** See [the migration doc](https://github.com/chanks/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
|
data/Gemfile
CHANGED
@@ -6,6 +6,7 @@ group :development, :test do
|
|
6
6
|
gem 'activerecord', :require => nil
|
7
7
|
gem 'sequel', :require => nil
|
8
8
|
gem 'connection_pool', :require => nil
|
9
|
+
gem 'pond', :require => nil
|
9
10
|
gem 'pg', :require => nil, :platform => :ruby
|
10
11
|
gem 'pg_jruby', :require => nil, :platform => :jruby
|
11
12
|
end
|
data/docs/managing_workers.md
CHANGED
@@ -25,6 +25,10 @@ If you don't want to burden your web processes with too much work and want to ru
|
|
25
25
|
# Or configure the number of workers:
|
26
26
|
QUE_WORKER_COUNT=8 rake que:work
|
27
27
|
|
28
|
+
Other options available via environment variables are `QUE_QUEUE` to determine which named queue jobs are pulled from, and `QUE_WAKE_INTERVAL` to determine how long workers will wait to poll again when there are no jobs available. For example, to run 2 workers that run jobs from the "other_queue" queue and wait a half-second between polls, you could do:
|
29
|
+
|
30
|
+
QUE_QUEUE=other_queue QUE_WORKER_COUNT=2 QUE_WAKE_INTERVAL=0.5 rake que:work
|
31
|
+
|
28
32
|
### Thread-Unsafe Application Code
|
29
33
|
|
30
34
|
If your application code is not thread-safe, you won't want any workers to be processing jobs while anything else is happening in the Ruby process. So, you'll want to turn the worker pool off by default:
|
data/docs/multiple_queues.md
CHANGED
@@ -8,6 +8,14 @@ Then you can set jobs to be enqueued in that queue specifically:
|
|
8
8
|
|
9
9
|
ProcessCreditCard.enqueue current_user.id, :queue => 'credit_cards'
|
10
10
|
|
11
|
+
# Or:
|
12
|
+
|
13
|
+
class ProcessCreditCard < Que::Job
|
14
|
+
# Set a default queue for this job class; this can be overridden by
|
15
|
+
# passing the :queue parameter to enqueue like above.
|
16
|
+
@queue = 'credit_cards'
|
17
|
+
end
|
18
|
+
|
11
19
|
In some cases, the ProcessCreditCard class may not be defined in the application that is enqueueing the job. In that case, you can specify the job class as a string:
|
12
20
|
|
13
21
|
Que.enqueue current_user.id, :job_class => 'ProcessCreditCard', :queue => 'credit_cards'
|
@@ -31,4 +31,20 @@ If you want to be able to use multithreading to run multiple jobs simultaneously
|
|
31
31
|
|
32
32
|
Be sure to pick your pool size carefully - if you use 10 for the size, you'll incur the overhead of having 10 connections open to Postgres even if you never use more than a couple of them.
|
33
33
|
|
34
|
-
|
34
|
+
The Pond gem doesn't have this drawback - it is very similar to ConnectionPool, but establishes connections lazily (add `gem 'pond'` to your Gemfile):
|
35
|
+
|
36
|
+
require 'uri'
|
37
|
+
require 'pg'
|
38
|
+
require 'pond'
|
39
|
+
|
40
|
+
uri = URI.parse(ENV['DATABASE_URL'])
|
41
|
+
|
42
|
+
Que.connection = Pond.new :maximum_size => 10 do
|
43
|
+
PG::Connection.open :host => uri.host,
|
44
|
+
:user => uri.user,
|
45
|
+
:password => uri.password,
|
46
|
+
:port => uri.port || 5432,
|
47
|
+
:dbname => uri.path[1..-1]
|
48
|
+
end
|
49
|
+
|
50
|
+
Please be aware that if you're using ActiveRecord or Sequel to manage your data, there's no reason for you to be using any of these methods - it's less efficient (unnecessary connections will waste memory on your database server) and you lose the reliability benefits of wrapping jobs in the same transactions as the rest of your data.
|
data/lib/que.rb
CHANGED
@@ -21,13 +21,15 @@ module Que
|
|
21
21
|
attr_writer :adapter, :log_formatter
|
22
22
|
|
23
23
|
def connection=(connection)
|
24
|
-
self.adapter =
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
self.adapter =
|
25
|
+
if connection.to_s == 'ActiveRecord'
|
26
|
+
Adapters::ActiveRecord.new
|
27
|
+
else
|
28
|
+
case connection.class.to_s
|
28
29
|
when 'Sequel::Postgres::Database' then Adapters::Sequel.new(connection)
|
29
30
|
when 'ConnectionPool' then Adapters::ConnectionPool.new(connection)
|
30
31
|
when 'PG::Connection' then Adapters::PG.new(connection)
|
32
|
+
when 'Pond' then Adapters::Pond.new(connection)
|
31
33
|
when 'NilClass' then connection
|
32
34
|
else raise "Que connection not recognized: #{connection.inspect}"
|
33
35
|
end
|
@@ -79,7 +81,7 @@ module Que
|
|
79
81
|
|
80
82
|
def log(data)
|
81
83
|
level = data.delete(:level) || :info
|
82
|
-
data = {:lib => 'que', :hostname => Socket.gethostname, :thread => Thread.current.object_id}.merge(data)
|
84
|
+
data = {:lib => 'que', :hostname => Socket.gethostname, :pid => Process.pid, :thread => Thread.current.object_id}.merge(data)
|
83
85
|
|
84
86
|
if logger && output = log_formatter.call(data)
|
85
87
|
logger.send level, output
|
data/lib/que/adapters/base.rb
CHANGED
data/lib/que/job.rb
CHANGED
@@ -40,10 +40,14 @@ module Que
|
|
40
40
|
|
41
41
|
attrs = {:job_class => job_class || to_s, :args => args}
|
42
42
|
|
43
|
+
warn "@default_run_at in #{to_s} has been deprecated and will be removed in Que version 1.0.0. Please use @run_at instead." if @default_run_at
|
44
|
+
|
43
45
|
if t = run_at || @run_at && @run_at.call || @default_run_at && @default_run_at.call
|
44
46
|
attrs[:run_at] = t
|
45
47
|
end
|
46
48
|
|
49
|
+
warn "@default_priority in #{to_s} has been deprecated and will be removed in Que version 1.0.0. Please use @priority instead." if @default_priority
|
50
|
+
|
47
51
|
if p = priority || @priority || @default_priority
|
48
52
|
attrs[:priority] = p
|
49
53
|
end
|
@@ -61,7 +65,10 @@ module Que
|
|
61
65
|
end
|
62
66
|
end
|
63
67
|
|
64
|
-
|
68
|
+
def queue(*args)
|
69
|
+
warn "#{to_s}.queue(*args) is deprecated and will be removed in Que version 1.0.0. Please use #{to_s}.enqueue(*args) instead."
|
70
|
+
enqueue(*args)
|
71
|
+
end
|
65
72
|
|
66
73
|
def run(*args)
|
67
74
|
new(:args => args).tap(&:_run)
|
@@ -98,7 +105,7 @@ module Que
|
|
98
105
|
begin
|
99
106
|
if job
|
100
107
|
count = job[:error_count].to_i + 1
|
101
|
-
interval = klass && klass.retry_interval || retry_interval
|
108
|
+
interval = klass && klass.respond_to?(:retry_interval) && klass.retry_interval || retry_interval
|
102
109
|
delay = interval.respond_to?(:call) ? interval.call(count) : interval
|
103
110
|
message = "#{error.message}\n#{error.backtrace.join("\n")}"
|
104
111
|
Que.execute :set_error, [count, delay, message] + job.values_at(:queue, :priority, :run_at, :job_id)
|
data/lib/que/rake_tasks.rb
CHANGED
@@ -12,7 +12,9 @@ namespace :que do
|
|
12
12
|
# the rake task in tasks/safe_shutdown.rb.
|
13
13
|
|
14
14
|
stop = false
|
15
|
-
|
15
|
+
%w( INT TERM ).each do |signal|
|
16
|
+
trap(signal) {stop = true}
|
17
|
+
end
|
16
18
|
|
17
19
|
at_exit do
|
18
20
|
$stdout.puts "Finishing Que's current jobs before exiting..."
|
@@ -26,9 +28,9 @@ namespace :que do
|
|
26
28
|
end
|
27
29
|
end
|
28
30
|
|
29
|
-
desc "
|
30
|
-
task :
|
31
|
-
Que.
|
31
|
+
desc "Migrate Que's job table to the most recent version (creating it if it doesn't exist)"
|
32
|
+
task :migrate => :environment do
|
33
|
+
Que.migrate!
|
32
34
|
end
|
33
35
|
|
34
36
|
desc "Drop Que's job table"
|
data/lib/que/version.rb
CHANGED
data/lib/que/worker.rb
CHANGED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'pond'
|
3
|
+
|
4
|
+
Que.connection = QUE_SPEC_POND = Pond.new &NEW_PG_CONNECTION
|
5
|
+
QUE_ADAPTERS[:pond] = Que.adapter
|
6
|
+
|
7
|
+
describe "Que using the Pond adapter" do
|
8
|
+
before { Que.adapter = QUE_ADAPTERS[:pond] }
|
9
|
+
|
10
|
+
it_behaves_like "a multi-threaded Que adapter"
|
11
|
+
|
12
|
+
it "should be able to tell when it's already in a transaction" do
|
13
|
+
Que.adapter.should_not be_in_transaction
|
14
|
+
QUE_SPEC_POND.checkout do |conn|
|
15
|
+
conn.async_exec "BEGIN"
|
16
|
+
Que.adapter.should be_in_transaction
|
17
|
+
conn.async_exec "COMMIT"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/spec/support/helpers.rb
CHANGED
data/spec/unit/enqueue_spec.rb
CHANGED
@@ -21,7 +21,7 @@ describe Que::Job, '.enqueue' do
|
|
21
21
|
|
22
22
|
it "should be aliased to .queue" do
|
23
23
|
DB[:que_jobs].count.should be 0
|
24
|
-
Que::Job.queue
|
24
|
+
suppress_warnings { Que::Job.queue }
|
25
25
|
DB[:que_jobs].count.should be 1
|
26
26
|
end
|
27
27
|
|
@@ -146,8 +146,10 @@ describe Que::Job, '.enqueue' do
|
|
146
146
|
end
|
147
147
|
|
148
148
|
DB[:que_jobs].count.should be 0
|
149
|
-
|
150
|
-
|
149
|
+
suppress_warnings do
|
150
|
+
OldDefaultPriorityJob.enqueue 1
|
151
|
+
OldDefaultPriorityJob.enqueue 1, :priority => 4
|
152
|
+
end
|
151
153
|
DB[:que_jobs].count.should be 2
|
152
154
|
|
153
155
|
first, second = DB[:que_jobs].order(:job_id).all
|
@@ -196,8 +198,10 @@ describe Que::Job, '.enqueue' do
|
|
196
198
|
end
|
197
199
|
|
198
200
|
DB[:que_jobs].count.should be 0
|
199
|
-
|
200
|
-
|
201
|
+
suppress_warnings do
|
202
|
+
OldDefaultRunAtJob.enqueue 1
|
203
|
+
OldDefaultRunAtJob.enqueue 1, :run_at => Time.now + 30
|
204
|
+
end
|
201
205
|
DB[:que_jobs].count.should be 2
|
202
206
|
|
203
207
|
first, second = DB[:que_jobs].order(:job_id).all
|
data/spec/unit/logging_spec.rb
CHANGED
@@ -8,6 +8,7 @@ describe "Logging" do
|
|
8
8
|
message = JSON.load($logger.messages.first)
|
9
9
|
message['lib'].should == 'que'
|
10
10
|
message['hostname'].should == Socket.gethostname
|
11
|
+
message['pid'].should == Process.pid
|
11
12
|
message['event'].should == 'blah'
|
12
13
|
message['source'].should == 4
|
13
14
|
message['thread'].should == Thread.current.object_id
|
data/spec/unit/work_spec.rb
CHANGED
@@ -349,5 +349,23 @@ describe Que::Job, '.work' do
|
|
349
349
|
job[:last_error].should =~ /uninitialized constant:? NonexistentClass/
|
350
350
|
job[:run_at].should be_within(3).of Time.now + 4
|
351
351
|
end
|
352
|
+
|
353
|
+
it "should throw an error properly if the corresponding job class doesn't descend from Que::Job" do
|
354
|
+
class J
|
355
|
+
def run(*args)
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
359
|
+
Que.enqueue :job_class => "J"
|
360
|
+
|
361
|
+
result = Que::Job.work
|
362
|
+
result[:event].should == :job_errored
|
363
|
+
result[:job][:job_class].should == 'J'
|
364
|
+
|
365
|
+
DB[:que_jobs].count.should be 1
|
366
|
+
job = DB[:que_jobs].first
|
367
|
+
job[:error_count].should be 1
|
368
|
+
job[:run_at].should be_within(3).of Time.now + 4
|
369
|
+
end
|
352
370
|
end
|
353
371
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Hanks
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-04-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -58,6 +58,7 @@ files:
|
|
58
58
|
- lib/que/adapters/base.rb
|
59
59
|
- lib/que/adapters/connection_pool.rb
|
60
60
|
- lib/que/adapters/pg.rb
|
61
|
+
- lib/que/adapters/pond.rb
|
61
62
|
- lib/que/adapters/sequel.rb
|
62
63
|
- lib/que/job.rb
|
63
64
|
- lib/que/migrations.rb
|
@@ -76,6 +77,7 @@ files:
|
|
76
77
|
- spec/adapters/active_record_spec.rb
|
77
78
|
- spec/adapters/connection_pool_spec.rb
|
78
79
|
- spec/adapters/pg_spec.rb
|
80
|
+
- spec/adapters/pond_spec.rb
|
79
81
|
- spec/adapters/sequel_spec.rb
|
80
82
|
- spec/gemfiles/Gemfile1
|
81
83
|
- spec/gemfiles/Gemfile2
|
@@ -128,6 +130,7 @@ test_files:
|
|
128
130
|
- spec/adapters/active_record_spec.rb
|
129
131
|
- spec/adapters/connection_pool_spec.rb
|
130
132
|
- spec/adapters/pg_spec.rb
|
133
|
+
- spec/adapters/pond_spec.rb
|
131
134
|
- spec/adapters/sequel_spec.rb
|
132
135
|
- spec/gemfiles/Gemfile1
|
133
136
|
- spec/gemfiles/Gemfile2
|