beanstalker 0.4.17 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/beanstalker.gemspec +3 -3
- data/lib/beanstalker/mapper.rb +33 -4
- data/lib/beanstalker/queue.rb +8 -6
- data/lib/beanstalker/worker.rb +48 -34
- metadata +6 -6
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.5.0
|
data/beanstalker.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{beanstalker}
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.5.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Gleb Pomykalov"]
|
12
|
-
s.date = %q{2011-
|
12
|
+
s.date = %q{2011-03-04}
|
13
13
|
s.description = %q{Beanstalker is a tool for executing long tasks in background in our rails application.}
|
14
14
|
s.email = %q{glebpom@gmail.com}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -30,7 +30,7 @@ Gem::Specification.new do |s|
|
|
30
30
|
]
|
31
31
|
s.homepage = %q{http://github.com/glebpom/beanstalker}
|
32
32
|
s.require_paths = ["lib"]
|
33
|
-
s.rubygems_version = %q{1.5.
|
33
|
+
s.rubygems_version = %q{1.5.2}
|
34
34
|
s.summary = %q{Beanstalker provides deep integration with Beanstalk. Fork from http://github.com/kristjan/async_observer}
|
35
35
|
|
36
36
|
if s.respond_to? :specification_version then
|
data/lib/beanstalker/mapper.rb
CHANGED
@@ -1,23 +1,52 @@
|
|
1
1
|
module Beanstalker
|
2
2
|
class Mapper
|
3
|
+
class Error < StandardError; end
|
4
|
+
class WithBlockNotPresent < Error; end
|
5
|
+
class NotAcceptable < Error; end
|
6
|
+
class KindUnknown < NotAcceptable; end
|
7
|
+
class TaskUnknown < NotAcceptable; end
|
8
|
+
|
3
9
|
def initialize(*args)
|
4
10
|
@mapping = {}
|
11
|
+
@current_kind = nil
|
5
12
|
args.each do |filename|
|
6
13
|
include_from_file(filename)
|
7
14
|
end
|
8
15
|
end
|
9
16
|
|
10
17
|
def include_from_file(filename)
|
11
|
-
puts "Loading #{filename}"
|
12
18
|
instance_eval(File.read(filename), filename)
|
13
19
|
end
|
14
20
|
|
15
21
|
def on(name, &block)
|
16
|
-
@
|
22
|
+
$logger.info "We are processing #{@current_kind}/#{name} now"
|
23
|
+
if @current_kind.nil?
|
24
|
+
raise WithBlockNotPresent, "Wrap #on calls with #with block to setup :kind of task in beanstalker_mapper.rb"
|
25
|
+
end
|
26
|
+
@mapping[@current_kind] ||= {}
|
27
|
+
@mapping[@current_kind][name.to_sym] = block
|
28
|
+
end
|
29
|
+
|
30
|
+
def with(kind, &block)
|
31
|
+
@current_kind = kind.to_sym
|
32
|
+
block.call
|
33
|
+
@current_kind = nil
|
17
34
|
end
|
18
35
|
|
19
|
-
def
|
20
|
-
@mapping[
|
36
|
+
def can_handle_kind?(kind)
|
37
|
+
@mapping && !! @mapping[kind.to_sym]
|
38
|
+
end
|
39
|
+
|
40
|
+
def method_for(kind, name)
|
41
|
+
kind_mapping = @mapping[kind.to_sym]
|
42
|
+
if kind_mapping.nil?
|
43
|
+
raise KindUnknown, "No handler for kind = #{kind.to_sym.inspect}"
|
44
|
+
end
|
45
|
+
task_mapping = kind_mapping[name.to_sym]
|
46
|
+
if task_mapping.nil?
|
47
|
+
raise TaskUnknown, "No handler for task = #{name.to_sym.inspect}"
|
48
|
+
end
|
49
|
+
task_mapping
|
21
50
|
end
|
22
51
|
end
|
23
52
|
end
|
data/lib/beanstalker/queue.rb
CHANGED
@@ -72,20 +72,22 @@ class << Beanstalker::Queue
|
|
72
72
|
|
73
73
|
if interpolator
|
74
74
|
code = packed = interpolator
|
75
|
-
else
|
75
|
+
else
|
76
76
|
code = gen(obj, sel, args)
|
77
77
|
packed = pkg(code, worker_opts, obj, sel)
|
78
78
|
end
|
79
|
-
|
79
|
+
|
80
80
|
RAILS_DEFAULT_LOGGER.info("put #{pri} #{code} to #{tube} with ttr #{ttr}")
|
81
81
|
put!(packed, pri, delay, ttr, tube)
|
82
82
|
end
|
83
83
|
|
84
84
|
def pkg(code, opts, obj, sel)
|
85
|
-
opts.merge(:
|
86
|
-
:
|
87
|
-
|
88
|
-
|
85
|
+
opts.merge(:kind => 'rails_beanstalker',
|
86
|
+
:data => {
|
87
|
+
:code => code,
|
88
|
+
:class => obj.is_a?(Class) ? obj.name : obj.class.to_s,
|
89
|
+
:method => sel.to_s
|
90
|
+
})
|
89
91
|
end
|
90
92
|
|
91
93
|
# Be careful not to pass in a selector that's not valid ruby source.
|
data/lib/beanstalker/worker.rb
CHANGED
@@ -147,11 +147,18 @@ class Beanstalker::Worker
|
|
147
147
|
|
148
148
|
def dispatch(job)
|
149
149
|
ActiveRecord::Base.verify_active_connections!
|
150
|
+
$logger.info "Got job: #{job.ybody.inspect}"
|
150
151
|
if rails_job?(job)
|
151
152
|
run_ao_job(job)
|
152
153
|
elsif mapped_job?(job)
|
153
154
|
run_mapped_job(job)
|
155
|
+
else
|
156
|
+
$logger.error "Job #{job.inspect} cannot be processed... deleteing"
|
157
|
+
job.delete
|
154
158
|
end
|
159
|
+
rescue Exception => e
|
160
|
+
$logger.error "Exception: #{e.inspect}... Bury job"
|
161
|
+
job.bury
|
155
162
|
end
|
156
163
|
|
157
164
|
def safe_dispatch(job)
|
@@ -182,24 +189,26 @@ class Beanstalker::Worker
|
|
182
189
|
|
183
190
|
def handle_error(job, ex)
|
184
191
|
custom_error_handler_ok = false
|
185
|
-
Daemonizer.logger.warn "Handling exception: #{ex.
|
186
|
-
|
187
|
-
if job
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
192
|
+
Daemonizer.logger.warn "Handling exception: #{ex.backtrace.join('\n')}, job = #{job.id}"
|
193
|
+
|
194
|
+
if rails_job?(job)
|
195
|
+
if job[:class]
|
196
|
+
klass = Object.const_get(job[:class])
|
197
|
+
error_handler = class_error_handler(klass)
|
198
|
+
if error_handler.is_a?(Proc)
|
199
|
+
Daemonizer.logger.info "Running custom error handler for class #{job[:class]}, job = #{job.id}"
|
200
|
+
error_handler.call(job, ex)
|
201
|
+
job_reserved = begin
|
202
|
+
job.stats['state'] == 'reserved'
|
203
|
+
rescue Beanstalk::NotFoundError
|
204
|
+
false
|
205
|
+
end
|
206
|
+
if job_reserved
|
207
|
+
Daemonizer.logger.info "Custom error handler for class #{job[:class]} didn't release job. job = #{job.id}"
|
208
|
+
else
|
209
|
+
Daemonizer.logger.info "Custom error handler for class #{job[:class]} released job. job = #{job.id}"
|
210
|
+
custom_error_handler_ok = true
|
211
|
+
end
|
203
212
|
end
|
204
213
|
end
|
205
214
|
end
|
@@ -252,27 +261,36 @@ class Beanstalker::Worker
|
|
252
261
|
end
|
253
262
|
|
254
263
|
def run_mapped_job(job)
|
255
|
-
|
264
|
+
job_body = job.ybody.stringify_keys
|
265
|
+
job_kind = job_body['kind']
|
266
|
+
job_data = job_body['data'].stringify_keys
|
267
|
+
job_method = job_data['method']
|
268
|
+
|
269
|
+
job_desc = "#{job_kind}/#{job_method}"
|
270
|
+
|
271
|
+
run_with_ruby_timeout_if_set(job_desc, job) do
|
256
272
|
t1 = Time.now
|
257
|
-
|
258
|
-
|
259
|
-
|
273
|
+
@map_job = @mapper && @mapper.method_for(job_kind, job_method)
|
274
|
+
if @map_job
|
275
|
+
@map_job.call(job_data['body'] || {})
|
276
|
+
logger.info "Finished. Job id=#{job.stats['id']}. Mapped from '#{job_desc}'. Time taken: #{(Time.now - t1).to_f} sec"
|
260
277
|
else
|
261
|
-
logger.error "Job id=#{job.stats['id']}. Mapping not found: '#{
|
278
|
+
logger.error "Job id=#{job.stats['id']}. Mapping not found: '#{job_desc}'. Releases #{job.stats['releases']}. Deleting"
|
262
279
|
end
|
263
280
|
job.delete
|
264
281
|
end
|
265
282
|
end
|
266
283
|
|
267
284
|
def run_ao_job(job)
|
268
|
-
|
285
|
+
job_data = job.ybody.stringify_keys['data']
|
286
|
+
code = job_data.stringify_keys['code']
|
287
|
+
run_with_ruby_timeout_if_set(code, job) do
|
269
288
|
t1 = Time.now
|
270
289
|
f = self.class.before_filter
|
271
290
|
statistics = job.stats.dup
|
272
|
-
code = job[:code]
|
273
291
|
can_run = f ? f.call(job) : true
|
274
292
|
if can_run
|
275
|
-
run_code(job)
|
293
|
+
run_code(job.id, code)
|
276
294
|
job.delete
|
277
295
|
logger.info "Finished. Job id=#{statistics['id']}. Code '#{code}'. Time taken: #{(Time.now - t1).to_f} sec"
|
278
296
|
else
|
@@ -281,20 +299,16 @@ class Beanstalker::Worker
|
|
281
299
|
end
|
282
300
|
end
|
283
301
|
|
284
|
-
def run_code(
|
285
|
-
eval(
|
302
|
+
def run_code(job_id, code)
|
303
|
+
eval(code, @top_binding, "(beanstalk job #{job_id})", 1)
|
286
304
|
end
|
287
305
|
|
288
306
|
def rails_job?(job)
|
289
|
-
|
307
|
+
job.ybody.stringify_keys['kind'].to_s == 'rails_beanstalker'
|
290
308
|
end
|
291
309
|
|
292
310
|
def mapped_job?(job)
|
293
|
-
|
294
|
-
end
|
295
|
-
|
296
|
-
def map_job(job)
|
297
|
-
@mapper && @mapper.method_for(job)
|
311
|
+
@mapper && @mapper.can_handle_kind?(job.ybody.stringify_keys['kind'])
|
298
312
|
end
|
299
313
|
|
300
314
|
def do_all_work
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beanstalker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 11
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 5
|
9
|
+
- 0
|
10
|
+
version: 0.5.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Gleb Pomykalov
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-03-04 00:00:00 +03:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -114,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
114
114
|
requirements: []
|
115
115
|
|
116
116
|
rubyforge_project:
|
117
|
-
rubygems_version: 1.5.
|
117
|
+
rubygems_version: 1.5.2
|
118
118
|
signing_key:
|
119
119
|
specification_version: 3
|
120
120
|
summary: Beanstalker provides deep integration with Beanstalk. Fork from http://github.com/kristjan/async_observer
|