langrove-plugin-resque 0.2 → 0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +1 -1
- data/README.md +37 -8
- data/langrove-plugin-resque.gemspec +1 -1
- data/lib/adaptor/resque_adaptor.rb +98 -21
- data/lib/handler/resque_handler.rb +3 -21
- data/lib/plugin/resque_queue.rb +20 -3
- data/spec/adaptor/resque_adaptor_spec.rb +89 -97
- data/spec/spec_helper.rb +4 -9
- metadata +2 -2
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
`Plugin::Resque`
|
2
2
|
===============
|
3
3
|
|
4
|
-
Integrates [Resque](https://github.com/defunkt/resque/) Worker and Job functionality into LanGrove. Resque
|
4
|
+
Integrates [Resque](https://github.com/defunkt/resque/) Worker and Job functionality into LanGrove. Resque uses [Redis](http://redis.io/documentation) under the hood.
|
5
5
|
|
6
6
|
`gem install langrove-plugin-resque`
|
7
7
|
|
@@ -10,14 +10,20 @@ Implements
|
|
10
10
|
|
11
11
|
* `Adaptor::ResqueAdaptor < LanGrove::Adaptor::Base` ...the worker (de-queue-er)
|
12
12
|
* `Handler::ResqueHandler < LanGrove::Handler::Base` ...the job
|
13
|
-
* `Plugin::ResqueQueue < LanGrove::Plugin::
|
14
|
-
|
13
|
+
* `Plugin::ResqueQueue < LanGrove::Plugin::Enqueuer` ...the en-queue-er
|
15
14
|
|
16
15
|
Supports
|
17
16
|
--------
|
18
17
|
|
19
|
-
* `LanGrove::Behaviour::
|
18
|
+
* `LanGrove::Behaviour::Enqueueable`
|
19
|
+
|
20
|
+
Important
|
21
|
+
---------
|
20
22
|
|
23
|
+
* Departs form traditional Resqu use.
|
24
|
+
* Each queue has a corresponding queue_failed queue.
|
25
|
+
* Uncaught exceptions from the Job are caught in the Worker and posted onto the _failed queue along with the original payload.
|
26
|
+
* There is no infrastructure to resubmit.
|
21
27
|
|
22
28
|
Examples
|
23
29
|
--------
|
@@ -30,18 +36,26 @@ Examples
|
|
30
36
|
|
31
37
|
:plugins:
|
32
38
|
:a_job_queue:
|
33
|
-
|
39
|
+
:class: ResqueQueue
|
34
40
|
:job: JobName
|
35
41
|
:queue: job_name
|
36
42
|
|
43
|
+
#
|
44
|
+
# Optionals (with defaults)
|
45
|
+
#
|
46
|
+
# Refers to the redis connection
|
47
|
+
#
|
48
|
+
:hostname: localhost
|
49
|
+
:port: 6397
|
50
|
+
|
37
51
|
...
|
38
52
|
|
39
53
|
:server:
|
40
54
|
:behavior:
|
41
|
-
:
|
55
|
+
:enqueueable:
|
42
56
|
:plugin: :a_job_queue
|
43
57
|
:at:
|
44
|
-
:handler_after_receive: :
|
58
|
+
:handler_after_receive: :enqueue_capsule
|
45
59
|
#
|
46
60
|
# will post the caspsule as payload
|
47
61
|
# onto the specified resque queue
|
@@ -61,7 +75,7 @@ Examples
|
|
61
75
|
|
62
76
|
:adaptor:
|
63
77
|
:class: ResqueAdaptor
|
64
|
-
:
|
78
|
+
:plugin: :a_job_queue
|
65
79
|
|
66
80
|
...
|
67
81
|
|
@@ -75,3 +89,18 @@ Examples
|
|
75
89
|
|
76
90
|
</pre>
|
77
91
|
|
92
|
+
|
93
|
+
Change Log
|
94
|
+
----------
|
95
|
+
|
96
|
+
### 2012-07-29
|
97
|
+
|
98
|
+
* Added support for sharing common plugin subconfig branch
|
99
|
+
* Added support for remote redis server.
|
100
|
+
|
101
|
+
### 2012-07-31
|
102
|
+
|
103
|
+
* Deferrable was renamed to enqueueable
|
104
|
+
* v0.3
|
105
|
+
|
106
|
+
|
@@ -2,7 +2,7 @@ $LOAD_PATH.unshift 'lib'
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |spec|
|
4
4
|
spec.name = %q{langrove-plugin-resque}
|
5
|
-
spec.version = '0.
|
5
|
+
spec.version = '0.3'
|
6
6
|
spec.date = DateTime.now.strftime( "%Y-%m-%d" )
|
7
7
|
spec.authors = ["Richard","","",""]
|
8
8
|
spec.email = %q{ops@clue.co.za}
|
@@ -35,24 +35,73 @@ module Adaptor
|
|
35
35
|
|
36
36
|
def initialize( root, config, deamon_name )
|
37
37
|
|
38
|
-
super
|
39
|
-
|
40
|
-
@type = :not_eventmachine
|
41
|
-
|
42
38
|
raise config_exception(
|
43
39
|
|
44
|
-
"#{self.class} requires :
|
40
|
+
"#{self.class} requires :plugin:"
|
45
41
|
|
46
42
|
) if (
|
47
43
|
|
48
44
|
config.nil? or
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
45
|
+
config[:plugin].nil?
|
46
|
+
|
47
|
+
)
|
48
|
+
|
49
|
+
super
|
50
|
+
|
51
|
+
raise config_exception(
|
52
|
+
|
53
|
+
"#{self.class} requires :plugin: to define :queue:"
|
54
|
+
|
55
|
+
) if (
|
53
56
|
|
57
|
+
config[:plugin][:queue].nil?
|
58
|
+
|
59
|
+
)
|
60
|
+
|
61
|
+
@queue = config[:plugin][:queue]
|
62
|
+
|
63
|
+
|
64
|
+
#
|
65
|
+
# Departure from traditional use of resque...
|
66
|
+
# - Each job has its own failed queue
|
67
|
+
# - Failed jobs are not resubmittable
|
68
|
+
# - The failure queue contains the exception
|
69
|
+
#
|
70
|
+
|
71
|
+
failed_queue = "#{@queue}_failed"
|
72
|
+
|
73
|
+
@failed_job_class = Object.const_set( failed_queue.camelize,
|
74
|
+
|
75
|
+
Class.new do
|
76
|
+
|
77
|
+
self.instance_variable_set(
|
78
|
+
|
79
|
+
:@queue, failed_queue
|
80
|
+
|
81
|
+
)
|
82
|
+
|
83
|
+
end
|
84
|
+
|
54
85
|
)
|
55
86
|
|
87
|
+
@hostname = config[:plugin][:hostname].nil? ?
|
88
|
+
|
89
|
+
'localhost' : config[:plugin][:hostname]
|
90
|
+
|
91
|
+
@port = config[:plugin][:port].nil? ?
|
92
|
+
|
93
|
+
6379 : config[:plugin][:port]
|
94
|
+
|
95
|
+
|
96
|
+
Resque.redis = Redis.new(
|
97
|
+
|
98
|
+
:host => @hostname,
|
99
|
+
:port => @port
|
100
|
+
|
101
|
+
)
|
102
|
+
|
103
|
+
@type = :not_eventmachine
|
104
|
+
|
56
105
|
end
|
57
106
|
|
58
107
|
def connector( params, &block )
|
@@ -62,7 +111,7 @@ module Adaptor
|
|
62
111
|
# one at a time,
|
63
112
|
# into the caller.
|
64
113
|
#
|
65
|
-
|
114
|
+
|
66
115
|
wait_for_job( @queue, &block )
|
67
116
|
|
68
117
|
end
|
@@ -74,21 +123,49 @@ module Adaptor
|
|
74
123
|
worker = Resque::Worker.new( queue )
|
75
124
|
|
76
125
|
worker.work do |job_spec|
|
77
|
-
|
78
|
-
new_job =
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
126
|
+
|
127
|
+
new_job = nil
|
128
|
+
|
129
|
+
begin
|
130
|
+
|
131
|
+
new_job = job_spec.payload_class.new
|
132
|
+
|
133
|
+
new_job.payload = job_spec.payload
|
134
|
+
|
135
|
+
#
|
136
|
+
# Job is yielded up the assignment pipeline to
|
137
|
+
# get access to all the local infrastructure.
|
138
|
+
#
|
139
|
+
|
140
|
+
yield new_job if block
|
141
|
+
|
142
|
+
rescue Exception => e
|
143
|
+
|
144
|
+
job_failed( job_spec, new_job, e )
|
145
|
+
|
146
|
+
end
|
88
147
|
|
89
148
|
end
|
90
149
|
|
91
150
|
end
|
151
|
+
|
152
|
+
def job_failed( spec, job, exception )
|
153
|
+
|
154
|
+
@logger.error "JOB FAILED ~ exception: #{exception.to_s}, payload: #{spec.payload}"
|
155
|
+
|
156
|
+
@root.trigger( job, :handler, :error )
|
157
|
+
|
158
|
+
Resque.enqueue(
|
159
|
+
|
160
|
+
@failed_job_class, Time.now, {
|
161
|
+
:exception => exception.inspect,
|
162
|
+
:payload => spec.payload
|
163
|
+
|
164
|
+
} )
|
165
|
+
|
166
|
+
job.error( exception )
|
167
|
+
|
168
|
+
end
|
92
169
|
|
93
170
|
end
|
94
171
|
|
@@ -4,7 +4,7 @@ module Handler
|
|
4
4
|
|
5
5
|
attr_accessor :payload
|
6
6
|
|
7
|
-
def
|
7
|
+
def work
|
8
8
|
|
9
9
|
#
|
10
10
|
# OVERRIDE
|
@@ -60,28 +60,10 @@ module Handler
|
|
60
60
|
|
61
61
|
|
62
62
|
#
|
63
|
-
# Call implementor
|
63
|
+
# Call implementor overridables
|
64
64
|
#
|
65
65
|
start
|
66
|
-
|
67
|
-
do_work
|
68
|
-
|
69
|
-
|
70
|
-
#
|
71
|
-
# Job was sent up the same assignment pipeline as
|
72
|
-
# socket handler,
|
73
|
-
#
|
74
|
-
# It has therefore been assigned into the server's
|
75
|
-
# @client collection
|
76
|
-
#
|
77
|
-
# It needs to be removed
|
78
|
-
#
|
79
|
-
I.notify( @logger, "need to disconnect from server here", [:vital] )
|
80
|
-
#
|
81
|
-
#
|
82
|
-
#
|
83
|
-
|
84
|
-
stop
|
66
|
+
work
|
85
67
|
|
86
68
|
end
|
87
69
|
|
data/lib/plugin/resque_queue.rb
CHANGED
@@ -12,7 +12,7 @@
|
|
12
12
|
|
13
13
|
module Plugin
|
14
14
|
|
15
|
-
class ResqueQueue < LanGrove::Plugin::
|
15
|
+
class ResqueQueue < LanGrove::Plugin::Enqueuer
|
16
16
|
|
17
17
|
def validate_config
|
18
18
|
|
@@ -48,11 +48,28 @@ module Plugin
|
|
48
48
|
end
|
49
49
|
|
50
50
|
)
|
51
|
+
|
52
|
+
|
53
|
+
@hostname = @config[:hostname].nil? ?
|
54
|
+
|
55
|
+
'localhost' : @config[:hostname]
|
56
|
+
|
57
|
+
@port = @config[:port].nil? ?
|
58
|
+
|
59
|
+
6379 : @config[:port]
|
60
|
+
|
61
|
+
|
62
|
+
Resque.redis = Redis.new(
|
63
|
+
|
64
|
+
:host => @hostname,
|
65
|
+
:port => @port
|
66
|
+
|
67
|
+
)
|
51
68
|
|
52
69
|
end
|
53
70
|
|
54
71
|
|
55
|
-
def
|
72
|
+
def enqueue_key( handler )
|
56
73
|
|
57
74
|
#
|
58
75
|
# Enqueue job with the handler.key
|
@@ -68,7 +85,7 @@ module Plugin
|
|
68
85
|
|
69
86
|
end
|
70
87
|
|
71
|
-
def
|
88
|
+
def enqueue_capsule( handler )
|
72
89
|
|
73
90
|
#
|
74
91
|
# Enqueue job with the handler.capsule
|
@@ -3,123 +3,119 @@ require 'spec_helper'
|
|
3
3
|
describe Adaptor::ResqueAdaptor do
|
4
4
|
|
5
5
|
before :all do
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
|
7
|
+
@daemon_name = 'deferror'
|
8
|
+
@plugin_name = :deferror
|
9
|
+
@queue_name = :resque_handler
|
10
10
|
|
11
11
|
end
|
12
12
|
|
13
13
|
|
14
14
|
before :each do
|
15
15
|
|
16
|
-
@
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
:queue => @queue_name
|
21
|
-
|
22
|
-
}
|
23
|
-
|
24
|
-
@connect_spec = {
|
25
|
-
|
26
|
-
:handler => {
|
27
|
-
|
28
|
-
:class => Handler::ResqueHandler,
|
29
|
-
|
30
|
-
:config => {}
|
31
|
-
|
32
|
-
}
|
33
|
-
|
16
|
+
CONFIG.plugins[@plugin_name] = {
|
17
|
+
|
18
|
+
:class => 'ResqueQueue',
|
19
|
+
:job => 'ResqueHandler',
|
20
|
+
:queue => @queue_name
|
21
|
+
|
34
22
|
}
|
35
|
-
|
23
|
+
|
36
24
|
end
|
37
25
|
|
38
26
|
subject {
|
39
27
|
|
40
28
|
Adaptor::ResqueAdaptor.new(
|
41
29
|
|
42
|
-
ROOT, @
|
30
|
+
ROOT, ROOT.config[:daemons][@daemon_name][:adaptor] , nil
|
43
31
|
|
44
32
|
)
|
45
33
|
|
46
34
|
}
|
35
|
+
|
47
36
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
37
|
+
context 'raises on' do
|
38
|
+
|
39
|
+
before :each do
|
40
|
+
|
41
|
+
CONFIG.create( @daemon_name, :adaptor, 'ResqueAdaptor' )
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'missing :plugin: config' do
|
46
|
+
|
47
|
+
expect {
|
57
48
|
|
58
|
-
|
59
|
-
|
60
|
-
LanGrove::DaemonConfigException,
|
49
|
+
Adaptor::ResqueAdaptor.new(
|
61
50
|
|
62
|
-
|
63
|
-
|
64
|
-
|
51
|
+
ROOT, ROOT.config[:daemons][@daemon_name][:adaptor], nil
|
52
|
+
|
53
|
+
)
|
54
|
+
|
55
|
+
}.to raise_error(
|
56
|
+
|
57
|
+
LanGrove::DaemonConfigException,
|
58
|
+
|
59
|
+
"Adaptor::ResqueAdaptor requires :plugin:"
|
60
|
+
|
61
|
+
)
|
62
|
+
end
|
65
63
|
|
66
64
|
end
|
67
65
|
|
68
66
|
context 'uses requeue to' do
|
69
67
|
|
70
68
|
before :each do
|
71
|
-
|
72
|
-
@
|
73
|
-
|
74
|
-
:
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
@worker = mock( 'Resque::Worker' )
|
87
|
-
|
88
|
-
@job = mock( 'Job' )
|
89
|
-
|
90
|
-
@payload = {}
|
91
|
-
|
92
|
-
Resque::Worker.should_receive(
|
93
|
-
|
94
|
-
:new
|
95
|
-
|
96
|
-
).and_return(
|
97
|
-
|
98
|
-
@worker
|
99
|
-
|
100
|
-
)
|
101
|
-
|
102
|
-
@worker.should_receive(
|
103
|
-
|
104
|
-
:work
|
105
|
-
|
106
|
-
).and_yield(
|
107
|
-
|
108
|
-
@job
|
109
|
-
|
69
|
+
|
70
|
+
CONFIG.create( @daemon_name, :adaptor, 'ResqueAdaptor', {
|
71
|
+
|
72
|
+
:plugin => @plugin_name
|
73
|
+
|
74
|
+
})
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'initializes redis with hostname and port' do
|
79
|
+
|
80
|
+
Redis.should_receive(
|
81
|
+
|
82
|
+
:new).with( {:host => "localhost", :port => 6397}
|
83
|
+
|
110
84
|
)
|
111
|
-
|
112
|
-
|
85
|
+
|
86
|
+
subject
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'get jobs from a queue' do
|
91
|
+
|
92
|
+
worker = mock( 'Resque::Worker' )
|
93
|
+
|
94
|
+
job = mock( 'Resque::Job' )
|
95
|
+
|
96
|
+
Resque::Worker.should_receive(
|
97
|
+
|
98
|
+
:new ).with( @queue_name
|
99
|
+
|
100
|
+
).and_return( worker )
|
101
|
+
|
102
|
+
worker.should_receive(
|
103
|
+
|
104
|
+
:work).and_yield( job
|
105
|
+
|
106
|
+
)
|
107
|
+
|
108
|
+
job.should_receive(
|
113
109
|
|
114
110
|
:payload_class
|
115
111
|
|
116
112
|
).and_return(
|
117
113
|
|
118
|
-
|
114
|
+
Handler::ResqueHandler
|
119
115
|
|
120
116
|
)
|
121
|
-
|
122
|
-
|
117
|
+
|
118
|
+
job.should_receive(
|
123
119
|
|
124
120
|
:payload
|
125
121
|
|
@@ -128,23 +124,19 @@ describe Adaptor::ResqueAdaptor do
|
|
128
124
|
@payload
|
129
125
|
|
130
126
|
)
|
131
|
-
|
132
|
-
end
|
133
|
-
|
134
|
-
it 'yields jobs' do
|
135
|
-
|
127
|
+
|
136
128
|
job = nil
|
137
|
-
|
138
|
-
subject.connector(
|
139
|
-
|
140
|
-
job =
|
141
|
-
|
129
|
+
|
130
|
+
subject.connector( {} ) do |next_job|
|
131
|
+
|
132
|
+
job = next_job
|
133
|
+
|
142
134
|
end
|
143
|
-
|
144
|
-
job.should be_a(
|
145
|
-
|
135
|
+
|
136
|
+
job.should be_a( Handler::ResqueHandler )
|
137
|
+
|
146
138
|
end
|
147
|
-
|
139
|
+
|
148
140
|
end
|
149
141
|
|
150
142
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,12 +1,7 @@
|
|
1
1
|
require 'langrove-plugin-resque'
|
2
2
|
|
3
|
-
RESQUE_DEFERRER =
|
4
|
-
|
5
|
-
:class => 'ResqueQueue',
|
6
|
-
:job => 'ResqueWorker',
|
7
|
-
:queue => :resque_queue
|
8
|
-
|
9
|
-
}
|
10
|
-
|
11
|
-
ROOT = LanGrove::FakeRoot.new
|
3
|
+
RESQUE_DEFERRER =
|
12
4
|
|
5
|
+
CONFIG = LanGrove::FakeConfig.new
|
6
|
+
LOG = LanGrove::FakeLogger
|
7
|
+
ROOT = LanGrove::Root::Base.new( nil, LOG, CONFIG )
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: langrove-plugin-resque
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.3'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2012-07-
|
15
|
+
date: 2012-07-31 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: langrove
|