beanpicker 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.rspec +2 -0
- data/README.markdown +186 -0
- data/Rakefile +47 -0
- data/beanpicker.gemspec +61 -0
- data/bin/combine +88 -0
- data/examples/sandwich.rb +46 -0
- data/lib/beanpicker.rb +268 -0
- data/lib/beanpicker/job_server.rb +374 -0
- data/lib/beanpicker/process.rb +23 -0
- data/lib/beanpicker/version.rb +9 -0
- data/spec/beanpicker_spec.rb +478 -0
- data/spec/spec_helper.rb +168 -0
- metadata +103 -0
@@ -0,0 +1,374 @@
|
|
1
|
+
require File.expand_path( File.join(File.dirname(__FILE__), "..", "beanpicker") )
|
2
|
+
require File.expand_path( File.join(File.dirname(__FILE__), "process") )
|
3
|
+
|
4
|
+
# Used by Beanpicker to kill childs and bury jobs
|
5
|
+
BEANPICKER_FORK = { :child_every => false, :child_master => false, :child_pid => 0 }
|
6
|
+
|
7
|
+
module Beanpicker
|
8
|
+
|
9
|
+
# Used by combine to run a server of jobs
|
10
|
+
class Server
|
11
|
+
|
12
|
+
include MsgLogger
|
13
|
+
|
14
|
+
# expect a list of files to load or pick from ARGV
|
15
|
+
def initialize(args=ARGV)
|
16
|
+
@args = args
|
17
|
+
end
|
18
|
+
|
19
|
+
# Create all workers and verify if exists any Worker or Worker::Child to start a loop.
|
20
|
+
# If no only exit
|
21
|
+
def run
|
22
|
+
debug "Hiring workers..."
|
23
|
+
for arg in @args
|
24
|
+
Worker.new arg
|
25
|
+
end
|
26
|
+
|
27
|
+
w = c = 0
|
28
|
+
for worker in Beanpicker::workers
|
29
|
+
w += 1
|
30
|
+
m = "Worker: #{worker.name} hired to do"
|
31
|
+
f = {}
|
32
|
+
for child in worker.childs
|
33
|
+
c += 1
|
34
|
+
f[child.job_name] ||= 0
|
35
|
+
f[child.job_name] += 1
|
36
|
+
end
|
37
|
+
for job in f.keys.sort
|
38
|
+
m << " #{job}##{f[job]}"
|
39
|
+
end
|
40
|
+
debug m
|
41
|
+
end
|
42
|
+
|
43
|
+
if w == 0
|
44
|
+
fatal ["NOBODY WANT TO WORK TO YOU!!",
|
45
|
+
"Have you specified a file?",
|
46
|
+
"e.g. #{$0} lib/my_file_with_jobs.rb"].join("\n")
|
47
|
+
exit
|
48
|
+
end
|
49
|
+
|
50
|
+
if c == 0
|
51
|
+
fatal ["ALL YOUR #{w} WORKER#{"S" if w > 1} ARE LAZY AND DON'T WANT TO DO ANYTHING!!",
|
52
|
+
"Have you specified a job in any of yous files?",
|
53
|
+
"e.g. job('no.lazy.worker') { |args| puts args[:my_money] }"].join("\n")
|
54
|
+
exit
|
55
|
+
end
|
56
|
+
|
57
|
+
debug "#{w} worker#{"s" if w > 1} hired to do #{c} job#{"s" if c > 1}, counting the money..."
|
58
|
+
|
59
|
+
loop { sleep 1 }
|
60
|
+
end
|
61
|
+
|
62
|
+
# Use the Beanpicker::log_handler to log messages
|
63
|
+
def log_handler
|
64
|
+
Beanpicker::log_handler
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
# Worker is the class used to create jobs
|
70
|
+
class Worker
|
71
|
+
|
72
|
+
include MsgLogger
|
73
|
+
|
74
|
+
# The name of worker. Pick from args[:name] or from the name of file(if passed)
|
75
|
+
attr_reader :name
|
76
|
+
# The list of childs
|
77
|
+
attr_reader :childs
|
78
|
+
|
79
|
+
# Accept a file and/or a block with jobs and add itself to workers list
|
80
|
+
def initialize(filepath=nil, args={}, &blk)
|
81
|
+
@childs = []
|
82
|
+
@name = args[:name] rescue nil
|
83
|
+
if filepath
|
84
|
+
@name = filepath.split(/[\\\/]/)[-1].gsub(/\.[^\.]+$/,'').split(/[_\.]/).map do |x|
|
85
|
+
x.capitalize
|
86
|
+
end.join if @name.nil?
|
87
|
+
|
88
|
+
begin
|
89
|
+
instance_eval File.read(filepath)
|
90
|
+
rescue => e
|
91
|
+
error Beanpicker::exception_message(e, "when loading file #{filepath}")
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
if block_given?
|
96
|
+
begin
|
97
|
+
instance_eval(&blk)
|
98
|
+
rescue => e
|
99
|
+
error Beanpicker::exception_message(e, "when evaluating block")
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
@name = "BeanpickerWorker without name" unless @name
|
104
|
+
Beanpicker::add_worker(self)
|
105
|
+
end
|
106
|
+
|
107
|
+
# Pass the job to Child::process and add the return to childs list
|
108
|
+
def job(name, args={}, &blk)
|
109
|
+
@childs << Child.process(name, args, self, &blk)
|
110
|
+
@childs.flatten!
|
111
|
+
end
|
112
|
+
|
113
|
+
# Use it own log_handler or global if not defined
|
114
|
+
def log_handler
|
115
|
+
@log_handler || Beanpicker::log_handler
|
116
|
+
end
|
117
|
+
|
118
|
+
# Shortcut to log_handler=
|
119
|
+
def log_file(f)
|
120
|
+
#without self call don't call log_handler= of this class Oo
|
121
|
+
self.log_handler = f
|
122
|
+
end
|
123
|
+
|
124
|
+
# Child is the class that handle the job.
|
125
|
+
# Every job can be one or more childs
|
126
|
+
class Child
|
127
|
+
|
128
|
+
include MsgLogger
|
129
|
+
|
130
|
+
# Process job.
|
131
|
+
# Use opts[:childs] or Beanpicker::default_childs_number to determine how many childs should be created
|
132
|
+
def self.process(job, opts={}, worker=nil, &blk)
|
133
|
+
(opts[:childs] || Beanpicker::default_childs_number).times.map do |i|
|
134
|
+
Child.new(job, opts, i, worker, &blk)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
|
139
|
+
# The name of job
|
140
|
+
attr_reader :job_name
|
141
|
+
# The number of job(generated by Child::process)
|
142
|
+
attr_reader :number
|
143
|
+
# Should fork every job?
|
144
|
+
attr_reader :fork_every
|
145
|
+
# Should fork the child process?
|
146
|
+
attr_reader :fork_master
|
147
|
+
# The pid of running job
|
148
|
+
attr_reader :fork_every_pid
|
149
|
+
# The pid of forked child process
|
150
|
+
attr_reader :fork_master_pid
|
151
|
+
# The default options merged with argument options
|
152
|
+
attr_reader :opts
|
153
|
+
# The Worker (father?) or nil
|
154
|
+
attr_reader :worker
|
155
|
+
|
156
|
+
# Create a new job, start a fork if @opts[:fork_master] and start the work
|
157
|
+
def initialize(job, opts={}, number=0, worker=nil, &blk)
|
158
|
+
@job_name = job
|
159
|
+
@opts = {
|
160
|
+
:childs => Beanpicker::default_childs_number,
|
161
|
+
:fork_every => Beanpicker::default_fork_every,
|
162
|
+
:fork_master => Beanpicker::default_fork_master
|
163
|
+
}.merge(opts)
|
164
|
+
@number = number
|
165
|
+
@blk = blk
|
166
|
+
@loop = nil
|
167
|
+
@beanstalk = Beanpicker::new_beanstalk
|
168
|
+
@run = true
|
169
|
+
@job = nil
|
170
|
+
@worker = worker
|
171
|
+
if @opts[:fork]
|
172
|
+
_fork_every = @opts[:fork].to_s == 'every'
|
173
|
+
_fork_master = @opts[:fork].to_s == 'master'
|
174
|
+
else
|
175
|
+
_fork_every = !!@opts[:fork_every]
|
176
|
+
_fork_master = !!@opts[:fork_master]
|
177
|
+
end
|
178
|
+
#really need self
|
179
|
+
self.log_handler = @opts[:log_file] unless @opts[:log_file].nil?
|
180
|
+
@fork_every = Beanpicker::fork_every.nil? ? _fork_every : Beanpicker::fork_every
|
181
|
+
@fork_master = Beanpicker::fork_master.nil? ? _fork_master : Beanpicker::fork_master
|
182
|
+
@fork_master_pid = nil
|
183
|
+
@fork_every_pid = nil
|
184
|
+
start_watch
|
185
|
+
start_loop
|
186
|
+
end
|
187
|
+
|
188
|
+
# Return the beanstalk connection(Child don't use the global connection)
|
189
|
+
def beanstalk
|
190
|
+
@beanstalk
|
191
|
+
end
|
192
|
+
|
193
|
+
# Watch the tube with job name and ignore all others
|
194
|
+
def start_watch
|
195
|
+
beanstalk.watch(@job_name)
|
196
|
+
beanstalk.list_tubes_watched.each do |server, tubes|
|
197
|
+
tubes.each { |tube| beanstalk.ignore(tube) unless tube == @job_name }
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
# Start the loop, fork if needed
|
202
|
+
def start_loop
|
203
|
+
return false if @loop and @loop.alive?
|
204
|
+
if @fork_master
|
205
|
+
fork_master_child_and_monitor
|
206
|
+
else
|
207
|
+
@loop = Thread.new(self) do |child|
|
208
|
+
work_loop(child)
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# call start_work passing itself(child) while @run
|
214
|
+
def work_loop(child)
|
215
|
+
start_work(child) while @run
|
216
|
+
end
|
217
|
+
|
218
|
+
# Here is all the magic :)
|
219
|
+
#
|
220
|
+
# Call fork and start the job and delete on end, bury the job if a error is raised
|
221
|
+
def start_work(child=self)
|
222
|
+
fork do
|
223
|
+
begin
|
224
|
+
@job = child.beanstalk.reserve
|
225
|
+
BEANPICKER_FORK[:job] = @job if BEANPICKER_FORK[:child_every]
|
226
|
+
data = @job.ybody
|
227
|
+
|
228
|
+
if not data.is_a?(Hash) or [:args, :next_jobs] - data.keys != []
|
229
|
+
data = { :args => data, :next_jobs => [] }
|
230
|
+
end
|
231
|
+
|
232
|
+
t=Time.now
|
233
|
+
debug "Running #{@job_name}##{@number} with args #{data[:args]}; next jobs #{data[:next_jobs]}"
|
234
|
+
r = @blk.call(data[:args].clone)
|
235
|
+
debug "Job #{@job_name}##{@number} finished in #{Time.now-t} seconds with return #{r}"
|
236
|
+
data[:args].merge!(r) if r.is_a?(Hash) and data[:args].is_a?(Hash)
|
237
|
+
|
238
|
+
@job.delete
|
239
|
+
|
240
|
+
Beanpicker.enqueue(data[:next_jobs], data[:args]) if r and not data[:next_jobs].empty?
|
241
|
+
rescue => e
|
242
|
+
fatal Beanpicker::exception_message(e, "in loop of #{@job_name}##{@number} with pid #{Process.pid}")
|
243
|
+
if BEANPICKER_FORK[:child_every]
|
244
|
+
exit
|
245
|
+
else
|
246
|
+
Thread.new(@job) { |j| j.bury rescue nil }
|
247
|
+
end
|
248
|
+
ensure
|
249
|
+
@job = nil
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# Fork the process if @fork_every and wait or only call the block
|
255
|
+
def fork(&blk)
|
256
|
+
if @fork_every
|
257
|
+
@fork_every_pid = pid = Kernel.fork do
|
258
|
+
BEANPICKER_FORK[:child_every] = true
|
259
|
+
Process.die_with_parent
|
260
|
+
at_exit_to_every_child_fork
|
261
|
+
$0 = "Beanpicker job child #{@job_name}##{@number} of #{Process.ppid}"
|
262
|
+
blk.call
|
263
|
+
end
|
264
|
+
if BEANPICKER_FORK[:child_master]
|
265
|
+
BEANPICKER_FORK[:child_pid] = pid
|
266
|
+
Process.waitpid pid
|
267
|
+
BEANPICKER_FORK[:child_pid] = nil
|
268
|
+
else
|
269
|
+
Process.waitpid pid
|
270
|
+
end
|
271
|
+
@fork_every_pid = nil
|
272
|
+
else
|
273
|
+
blk.call
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
# The child still working?
|
278
|
+
def running?
|
279
|
+
@run
|
280
|
+
end
|
281
|
+
|
282
|
+
# Stop running, kill the thread and kill master os every process
|
283
|
+
def die!
|
284
|
+
@run = false
|
285
|
+
@loop.kill if @loop and @loop.alive?
|
286
|
+
|
287
|
+
kill_pid = nil
|
288
|
+
if @fork_master
|
289
|
+
kill_pid = @fork_master_pid
|
290
|
+
elsif @fork_every
|
291
|
+
kill_pid = @fork_every_pid
|
292
|
+
end
|
293
|
+
|
294
|
+
if kill_pid and kill_pid.is_a?(Integer) and Process.running?(kill_pid)
|
295
|
+
debug "Killing child with pid #{kill_pid}"
|
296
|
+
Process.kill "TERM", kill_pid
|
297
|
+
end
|
298
|
+
|
299
|
+
end
|
300
|
+
|
301
|
+
# Crete a new fork, change the name and create a thread to restart the fork if it die
|
302
|
+
#
|
303
|
+
# Called by start_loop when fork_master
|
304
|
+
def fork_master_child_and_monitor
|
305
|
+
@fork_master_pid = Kernel.fork do
|
306
|
+
at_exit_to_master_child_fork
|
307
|
+
Process.die_with_parent
|
308
|
+
BEANPICKER_FORK[:child_master] = true
|
309
|
+
$0 = "Beanpicker master child #{@job_name}##{@number}"
|
310
|
+
work_loop(self)
|
311
|
+
end
|
312
|
+
@loop = Thread.new(self) do |child|
|
313
|
+
Process.waitpid @fork_master_pid
|
314
|
+
child.fork_master_child_and_monitor if child.running?
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
318
|
+
# Create a at_exit to kill the child(if any)
|
319
|
+
#
|
320
|
+
# Used by fork_master_child_and_monitor
|
321
|
+
def at_exit_to_master_child_fork
|
322
|
+
at_exit do
|
323
|
+
pid = BEANPICKER_FORK[:child_pid]
|
324
|
+
if pid and pid > 0
|
325
|
+
if Process.running?(pid)
|
326
|
+
Process.kill "TERM", pid
|
327
|
+
sleep 0.1
|
328
|
+
if Process.running?(pid)
|
329
|
+
sleep 2
|
330
|
+
Process.kill "KILL", pid if Process.running?(pid)
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
334
|
+
Kernel.exit!
|
335
|
+
end
|
336
|
+
end
|
337
|
+
|
338
|
+
# Create a at_exit to bury the job(if any) and exit
|
339
|
+
#
|
340
|
+
# Used by fork
|
341
|
+
def at_exit_to_every_child_fork
|
342
|
+
at_exit do
|
343
|
+
Thread.new do
|
344
|
+
sleep 1
|
345
|
+
Kernel.exit!
|
346
|
+
end
|
347
|
+
BEANPICKER_FORK[:job].bury rescue nil if BEANPICKER_FORK[:job]
|
348
|
+
end
|
349
|
+
end
|
350
|
+
|
351
|
+
# Return the own log_handler, or the Worker log_handler or the global log_handler
|
352
|
+
def log_handler
|
353
|
+
#'@log_handler || ' go to worker/global log_handler even if @log_handler is defined
|
354
|
+
defined?(@log_handler) ? @log_handler : @worker.nil? ? Beanpicker::log_handler : @worker.log_handler
|
355
|
+
end
|
356
|
+
|
357
|
+
end
|
358
|
+
|
359
|
+
end
|
360
|
+
|
361
|
+
end
|
362
|
+
|
363
|
+
# kill childs or stop jobs
|
364
|
+
at_exit do
|
365
|
+
if not BEANPICKER_FORK[:child_master] and not BEANPICKER_FORK[:child_every]
|
366
|
+
Beanpicker::debug "Laying off workers..." if Beanpicker::workers.count > 0
|
367
|
+
Beanpicker::stop_workers
|
368
|
+
end
|
369
|
+
end
|
370
|
+
|
371
|
+
# hide errors throwed with ctrl+c
|
372
|
+
trap("INT") {
|
373
|
+
exit
|
374
|
+
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Process Premium Version + Crack + Serial + two new methods
|
2
|
+
module Process
|
3
|
+
|
4
|
+
# Verify if a process is running
|
5
|
+
def self.running?(pid)
|
6
|
+
begin
|
7
|
+
Process.kill 0, pid
|
8
|
+
return true
|
9
|
+
rescue Errno::ESRCH, Errno::EPERM
|
10
|
+
return false
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Create a thread to kill the current process if the parent process die
|
15
|
+
def self.die_with_parent
|
16
|
+
Thread.new do
|
17
|
+
while Process.running?(Process.ppid)
|
18
|
+
sleep 1
|
19
|
+
end
|
20
|
+
Kernel.exit
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,478 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Beanpicker do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
Beanpicker::default_fork_every = false
|
7
|
+
Beanpicker::default_fork_master = false
|
8
|
+
end
|
9
|
+
|
10
|
+
before(:each) do
|
11
|
+
@beanstalk = Beanpicker::beanstalk
|
12
|
+
@ms = { :values => {}, :methods => [:default_fork_every,
|
13
|
+
:default_fork_master,
|
14
|
+
:fork_every,
|
15
|
+
:fork_master,
|
16
|
+
:default_childs_number] }
|
17
|
+
for m in @ms[:methods]
|
18
|
+
@ms[:values][m] = Beanpicker.send(m)
|
19
|
+
end
|
20
|
+
|
21
|
+
Thread.stub!(:new)
|
22
|
+
Kernel.stub!(:fork)
|
23
|
+
Process.stub!(:waitpid)
|
24
|
+
end
|
25
|
+
|
26
|
+
after(:each) do
|
27
|
+
for m in @ms[:methods]
|
28
|
+
Beanpicker.send("#{m}=", @ms[:values][m])
|
29
|
+
end
|
30
|
+
Beanpicker::workers.replace []
|
31
|
+
end
|
32
|
+
|
33
|
+
context "Getters and Setters" do
|
34
|
+
|
35
|
+
describe "log_handler" do
|
36
|
+
|
37
|
+
it "should use STDOUT by default" do
|
38
|
+
Logger.should_receive(:new).with(STDOUT).once
|
39
|
+
Beanpicker::log_handler
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should use string as a argument to Logger" do
|
43
|
+
Logger.should_receive(:new).with("/dev/null").once
|
44
|
+
Beanpicker::log_handler = "/dev/null"
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should use IO as a argument to Logger" do
|
48
|
+
Logger.should_receive(:new).with(STDERR).once
|
49
|
+
Beanpicker::log_handler = STDERR
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should use anything that isn't IO/String and respond to 5 log methods as a logger" do
|
53
|
+
l=Logger.new(STDOUT)
|
54
|
+
Beanpicker::log_handler = l
|
55
|
+
Beanpicker::log_handler.should be_equal(l)
|
56
|
+
end
|
57
|
+
|
58
|
+
it "should not use anything that ins't IO/String and don't respond to 5 log methods as a logger" do
|
59
|
+
Beanpicker.should_receive(:error).once
|
60
|
+
l = Beanpicker::log_handler
|
61
|
+
Beanpicker::log_handler = :foo
|
62
|
+
Beanpicker::log_handler.should be_equal(l)
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
it "default_pri" do
|
68
|
+
Beanpicker::default_pri = 10
|
69
|
+
Beanpicker::default_pri.should be_equal(10)
|
70
|
+
end
|
71
|
+
|
72
|
+
it "default_delay" do
|
73
|
+
Beanpicker::default_delay = 10
|
74
|
+
Beanpicker::default_delay.should be_equal(10)
|
75
|
+
end
|
76
|
+
|
77
|
+
it "default_ttr" do
|
78
|
+
Beanpicker::default_ttr = 10
|
79
|
+
Beanpicker::default_ttr.should be_equal(10)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "default_childs_number" do
|
83
|
+
Beanpicker::default_childs_number = 10
|
84
|
+
Beanpicker::default_childs_number.should be_equal(10)
|
85
|
+
end
|
86
|
+
|
87
|
+
it "default_fork_every" do
|
88
|
+
Beanpicker::default_fork_every = true
|
89
|
+
Beanpicker::default_fork_every.should be_true
|
90
|
+
Beanpicker::default_fork_every = false
|
91
|
+
Beanpicker::default_fork_every.should be_false
|
92
|
+
end
|
93
|
+
|
94
|
+
it "default_fork_master" do
|
95
|
+
Beanpicker::default_fork_master = true
|
96
|
+
Beanpicker::default_fork_master.should be_true
|
97
|
+
Beanpicker::default_fork_master = false
|
98
|
+
Beanpicker::default_fork_master.should be_false
|
99
|
+
end
|
100
|
+
|
101
|
+
it "fork_every" do
|
102
|
+
Beanpicker::fork_every = true
|
103
|
+
Beanpicker::fork_every.should be_true
|
104
|
+
Beanpicker::fork_every = false
|
105
|
+
Beanpicker::fork_every.should be_false
|
106
|
+
end
|
107
|
+
|
108
|
+
it "fork_master" do
|
109
|
+
Beanpicker::fork_master = true
|
110
|
+
Beanpicker::fork_master.should be_true
|
111
|
+
Beanpicker::fork_master = false
|
112
|
+
Beanpicker::fork_master.should be_false
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
117
|
+
describe "enqueue" do
|
118
|
+
|
119
|
+
it "should use job name as tube and send via yput" do
|
120
|
+
@beanstalk.should_receive(:use).once.with("foo.bar")
|
121
|
+
@beanstalk.should_receive(:yput).once.with({ :args => { :foo => :bar }, :next_jobs => [] }, 1, 2, 3)
|
122
|
+
Beanpicker.enqueue("foo.bar", { :foo => :bar }, { :pri => 1, :delay => 2, :ttr => 3 })
|
123
|
+
end
|
124
|
+
|
125
|
+
it "should use default pri, delay and ttr and argument as a empty hash" do
|
126
|
+
@beanstalk.should_receive(:yput).once.with({ :args => {}, :next_jobs => [] },
|
127
|
+
Beanpicker::default_pri,
|
128
|
+
Beanpicker::default_delay,
|
129
|
+
Beanpicker::default_ttr)
|
130
|
+
Beanpicker::enqueue("foo.bar")
|
131
|
+
end
|
132
|
+
|
133
|
+
it "should understand chain jobs" do
|
134
|
+
@beanstalk.should_receive(:yput).once.with({ :args => {}, :next_jobs => ["foo.bar2", "foo.bar3"]}, 1, 2, 3)
|
135
|
+
@beanstalk.should_receive(:use).once.with("foo.bar")
|
136
|
+
Beanpicker::enqueue(["foo.bar", "foo.bar2", "foo.bar3"], {},
|
137
|
+
{ :pri => 1, :delay => 2, :ttr => 3 })
|
138
|
+
end
|
139
|
+
|
140
|
+
it "should handle connection error and retry" do
|
141
|
+
Beanpicker::beanstalk.should be_equal(@beanstalk)
|
142
|
+
@beanstalk.close
|
143
|
+
b = Beanpicker::new_beanstalk
|
144
|
+
TCPSocket.fail_times(1)
|
145
|
+
Beanpicker.should_receive(:new_beanstalk).and_return(b)
|
146
|
+
Beanpicker.should_receive(:exception_message).once.with(an_instance_of(Beanstalk::NotConnected), an_instance_of(String))
|
147
|
+
Beanpicker.should_receive(:error).once
|
148
|
+
Beanpicker.enqueue("foo.bar")
|
149
|
+
end
|
150
|
+
|
151
|
+
it "should raise a exeption if a retry don't work" do
|
152
|
+
@beanstalk.close
|
153
|
+
TCPSocket.fail_times 2
|
154
|
+
Beanpicker.should_receive(:error).once
|
155
|
+
expect { Beanpicker::enqueue("foo.bar") }.should raise_error(Beanstalk::NotConnected)
|
156
|
+
TCPSocket.fail_times 0
|
157
|
+
end
|
158
|
+
|
159
|
+
|
160
|
+
end
|
161
|
+
|
162
|
+
context "new beanstalk" do
|
163
|
+
|
164
|
+
it "new_beanstalk should create a new instance of Beanstalk::Pool every time" do
|
165
|
+
Beanpicker::new_beanstalk.should_not be_equal(Beanpicker::new_beanstalk)
|
166
|
+
end
|
167
|
+
|
168
|
+
it "new_beanstalk_should use beanstalk_urls and return [localhost:11300] by default" do
|
169
|
+
Beanpicker::beanstalk_urls.should include("localhost:11300")
|
170
|
+
Beanpicker.should_receive(:beanstalk_urls).and_return(["localhost:11300"])
|
171
|
+
Beanpicker::new_beanstalk
|
172
|
+
end
|
173
|
+
|
174
|
+
it "beanstalk_urls should read ENV['BEANSTALK_URL'] and ENV['BEANSTALK_URLS']" do
|
175
|
+
ENV.should_receive(:[]).with("BEANSTALK_URL").and_return("localhost:3000")
|
176
|
+
ENV.should_receive(:[]).with("BEANSTALK_URLS").and_return("localhost:1500,beanstalk://www.foo.bar.net:4500")
|
177
|
+
urls = Beanpicker::beanstalk_urls
|
178
|
+
urls.should include("localhost:3000")
|
179
|
+
urls.should include("localhost:1500")
|
180
|
+
urls.should include("www.foo.bar.net:4500")
|
181
|
+
urls.should_not include("localhost:11300")
|
182
|
+
end
|
183
|
+
|
184
|
+
end
|
185
|
+
|
186
|
+
it "add_worker should add a worker(duh!)" do
|
187
|
+
expect { Beanpicker::add_worker 1 }.should change(Beanpicker::workers, :size).by(1)
|
188
|
+
end
|
189
|
+
|
190
|
+
it "stop_workers should kill childs of workers" do
|
191
|
+
child = mock(Beanpicker::Worker::Child)
|
192
|
+
worker = mock(Beanpicker::Worker)
|
193
|
+
worker.should_receive(:childs).and_return([child])
|
194
|
+
child.should_receive(:die!)
|
195
|
+
Beanpicker::workers.should_receive(:clear)
|
196
|
+
Beanpicker::add_worker worker
|
197
|
+
Beanpicker::stop_workers
|
198
|
+
end
|
199
|
+
|
200
|
+
describe Beanpicker::Worker do
|
201
|
+
|
202
|
+
it "should handle a error when loading a file" do
|
203
|
+
pending "don't know how make"
|
204
|
+
Beanpicker::Worker.should_receive(:error)
|
205
|
+
Beanpicker::Worker.new("/foo/bar/lol.rb")
|
206
|
+
end
|
207
|
+
|
208
|
+
it "should handle a error when evaluating a block" do
|
209
|
+
pending "don't know how make"
|
210
|
+
Beanpicker::Worker.new do
|
211
|
+
foo
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
it "should add itself to workers list" do
|
216
|
+
expect { Beanpicker::Worker.new }.should change(Beanpicker::workers, :count).by(1)
|
217
|
+
Beanpicker::workers.should include(Beanpicker::Worker.new)
|
218
|
+
end
|
219
|
+
|
220
|
+
it "job should call Child.process" do
|
221
|
+
w=Beanpicker::Worker.new
|
222
|
+
Beanpicker::Worker::Child.should_receive(:process).and_return([1])
|
223
|
+
expect { w.job("foo.bar") }.should change(w.childs, :count).by(1)
|
224
|
+
end
|
225
|
+
|
226
|
+
context "log_handler" do
|
227
|
+
|
228
|
+
it "should use global log_handler if haven't it own defined" do
|
229
|
+
l = Logger.new(STDOUT)
|
230
|
+
Beanpicker.should_receive(:log_handler).twice.and_return(l)
|
231
|
+
l.should_receive(:debug)
|
232
|
+
w = Beanpicker::Worker.new
|
233
|
+
w.log_handler.should be_equal(l)
|
234
|
+
w.debug("foo!")
|
235
|
+
end
|
236
|
+
|
237
|
+
it "should use it own log_handler if defined" do
|
238
|
+
w=Beanpicker::Worker.new
|
239
|
+
w.should_receive(:debug)
|
240
|
+
w.log_handler = STDOUT
|
241
|
+
w.debug("foo!")
|
242
|
+
end
|
243
|
+
|
244
|
+
it "should use log_file as a mirror to log_handler" do
|
245
|
+
w=Beanpicker::Worker.new
|
246
|
+
w.should_receive(:log_handler=)
|
247
|
+
w.log_file 'foo'
|
248
|
+
end
|
249
|
+
|
250
|
+
end
|
251
|
+
|
252
|
+
end
|
253
|
+
|
254
|
+
describe Beanpicker::Worker::Child do
|
255
|
+
|
256
|
+
|
257
|
+
def c(*a, &blk)
|
258
|
+
Beanpicker::Worker::Child.new(*a, &blk)
|
259
|
+
end
|
260
|
+
|
261
|
+
context "Child.process"do
|
262
|
+
|
263
|
+
it "should create one child by default" do
|
264
|
+
Beanpicker::Worker::Child.process("foo").count.should be_equal(1)
|
265
|
+
end
|
266
|
+
|
267
|
+
it "should respect default_childs_number if no option :childs is passed" do
|
268
|
+
Beanpicker::default_childs_number = 3
|
269
|
+
Beanpicker::Worker::Child.process("foo").count.should be_equal(3)
|
270
|
+
end
|
271
|
+
|
272
|
+
it "should respect :childs if passed" do
|
273
|
+
Beanpicker::Worker::Child.process("foo", { :childs => 3 }).count.should be_equal(3)
|
274
|
+
end
|
275
|
+
|
276
|
+
end
|
277
|
+
|
278
|
+
context "new" do
|
279
|
+
|
280
|
+
it "should create it own beanstalk instance" do
|
281
|
+
b = Beanpicker::new_beanstalk
|
282
|
+
Beanpicker.should_receive(:new_beanstalk).and_return(b)
|
283
|
+
c("foo", {}, 1)
|
284
|
+
end
|
285
|
+
|
286
|
+
it "should define job_name, number and opts and worker" do
|
287
|
+
j = "foo"
|
288
|
+
n = 1
|
289
|
+
o = { :foo => :bar }
|
290
|
+
w = mock("Worker")
|
291
|
+
child = c(j, o, n, w)
|
292
|
+
child.job_name.should be_equal(j)
|
293
|
+
child.number.should be_equal(n)
|
294
|
+
child.opts.should_not be_equal(o)
|
295
|
+
child.opts[:foo].should be_equal(o[:foo])
|
296
|
+
child.worker.should be_equal(w)
|
297
|
+
end
|
298
|
+
|
299
|
+
it "should understand :fork_every and :fork_master" do
|
300
|
+
c = c("foo", { :fork_every => true, :fork_master => true })
|
301
|
+
c.fork_every.should be_true
|
302
|
+
c.fork_master.should be_true
|
303
|
+
end
|
304
|
+
|
305
|
+
it "should overwrite :fork_every and :fork_master with :fork" do
|
306
|
+
c1 = c("foo", { :fork_every => false, :fork_master => true, :fork => :every })
|
307
|
+
c2 = c("foo", { :fork_every => true, :fork_master => false, :fork => :master })
|
308
|
+
c1.fork_every.should be_true
|
309
|
+
c1.fork_master.should be_false
|
310
|
+
c2.fork_every.should be_false
|
311
|
+
c2.fork_master.should be_true
|
312
|
+
end
|
313
|
+
|
314
|
+
it "should create it own log_handler if :log_file is passed and is a String or a IO" do
|
315
|
+
l = Logger.new STDOUT
|
316
|
+
Beanpicker.should_receive(:log_handler).exactly(0).times
|
317
|
+
c1=c("foo", { :log_file => l })
|
318
|
+
c2=c("foo", { :log_file => "/dev/null" })
|
319
|
+
c1.log_handler.should be_equal(l)
|
320
|
+
c2.log_handler
|
321
|
+
end
|
322
|
+
|
323
|
+
end
|
324
|
+
|
325
|
+
it "should watch only the tube with job_name" do
|
326
|
+
c = c("foo")
|
327
|
+
b = c.beanstalk
|
328
|
+
b.watch("bar")
|
329
|
+
b.should_receive(:watch).with("foo")
|
330
|
+
b.should_receive(:ignore).with("bar")
|
331
|
+
c.start_watch
|
332
|
+
end
|
333
|
+
|
334
|
+
context "start_loop" do
|
335
|
+
|
336
|
+
it "should call fork_master_child_and_monitor if have fork_master" do
|
337
|
+
c=c("foo", :fork_master => true)
|
338
|
+
c.should_receive(:fork_master_child_and_monitor)
|
339
|
+
c.start_loop
|
340
|
+
end
|
341
|
+
|
342
|
+
it "should call Thread.new if haven't fork_master" do
|
343
|
+
Thread.should_receive(:new)
|
344
|
+
c=c("foo", :fork_master => false)
|
345
|
+
end
|
346
|
+
|
347
|
+
end
|
348
|
+
|
349
|
+
context "fork" do
|
350
|
+
|
351
|
+
it "should call Kernel.fork if have fork_every" do
|
352
|
+
c=c("foo", :fork_every => true)
|
353
|
+
Kernel.should_receive(:fork).once.and_return(0)
|
354
|
+
c.fork {}
|
355
|
+
end
|
356
|
+
|
357
|
+
it "should call only block if haven't fork_every" do
|
358
|
+
c=c("foo", :fork_every => false)
|
359
|
+
b = proc {}
|
360
|
+
Kernel.should_receive(:fork).exactly(0).times
|
361
|
+
b.should_receive(:call).once
|
362
|
+
c.fork(&b)
|
363
|
+
end
|
364
|
+
|
365
|
+
end
|
366
|
+
|
367
|
+
context "start_work" do
|
368
|
+
|
369
|
+
before(:each) do
|
370
|
+
@beanstalk.yput({ :args => { :foo => :bar }})
|
371
|
+
@job = @beanstalk.reserve
|
372
|
+
end
|
373
|
+
|
374
|
+
it "should reserve and delete a job" do
|
375
|
+
c=c("foo"){}
|
376
|
+
c.beanstalk.should_receive(:reserve).once.and_return(@job)
|
377
|
+
@job.should_receive(:ybody).once.and_return({})
|
378
|
+
@job.should_receive(:delete).once
|
379
|
+
c.should_receive(:fatal).exactly(0).times
|
380
|
+
c.start_work
|
381
|
+
end
|
382
|
+
|
383
|
+
it "should bury the job if got a eror" do
|
384
|
+
c=c("foo"){ raise RuntimeError }
|
385
|
+
@job.should_receive(:delete).exactly(0).times
|
386
|
+
@job.should_receive(:bury).once
|
387
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
388
|
+
Thread.should_receive(:new).once.with(@job).and_yield(@job)
|
389
|
+
c.start_work
|
390
|
+
end
|
391
|
+
|
392
|
+
it "should not do chain jobs if receive false from block" do
|
393
|
+
c=c("foo"){ false }
|
394
|
+
@job.ybody[:next_jobs] = ["foo"]
|
395
|
+
Beanpicker.should_receive(:enqueue).exactly(0).times
|
396
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
397
|
+
c.start_work
|
398
|
+
end
|
399
|
+
|
400
|
+
it "should not do chain jobs if receive nil from block" do
|
401
|
+
c=c("foo"){ nil }
|
402
|
+
@job.ybody[:next_jobs] = ["foo"]
|
403
|
+
Beanpicker.should_receive(:enqueue).exactly(0).times
|
404
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
405
|
+
c.start_work
|
406
|
+
end
|
407
|
+
|
408
|
+
it "should do chain jobs if receive a positive return from block" do
|
409
|
+
c=c("foo"){ true }
|
410
|
+
@job.ybody[:next_jobs] = ["foo"]
|
411
|
+
Beanpicker.should_receive(:enqueue).once.with(@job.ybody[:next_jobs], @job.ybody[:args])
|
412
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
413
|
+
c.start_work
|
414
|
+
end
|
415
|
+
|
416
|
+
it "should do chain jobs with merged args if receive a hash from block" do
|
417
|
+
c=c("foo"){ { :lol => :bar } }
|
418
|
+
@job.ybody[:next_jobs] = ["foo"]
|
419
|
+
Beanpicker.should_receive(:enqueue).once.with(@job.ybody[:next_jobs], hash_including(@job.ybody[:args].merge({ :lol => :bar })))
|
420
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
421
|
+
c.start_work
|
422
|
+
end
|
423
|
+
|
424
|
+
end
|
425
|
+
|
426
|
+
context "die!" do
|
427
|
+
|
428
|
+
it "should call Process.kill if have a forked child" do
|
429
|
+
pending "with die! call it two times Oo"
|
430
|
+
c=c("foo", { :fork_every => true }){}
|
431
|
+
Kernel.should_receive(:fork).once.and_return(1234)
|
432
|
+
Process.should_receive(:waitpid).once.with { c.die!; 1234 }
|
433
|
+
Process.should_receive(:running?).once.with(1234).and_return(true)
|
434
|
+
Process.should_receive(:kill).once.with("TERM", 1234)
|
435
|
+
c.fork
|
436
|
+
end
|
437
|
+
|
438
|
+
it "should not call Process.kill if haven't a forked child" do
|
439
|
+
c=c("foo", { :fork_every => false }){}
|
440
|
+
Process.should_receive(:running?).exactly(0).times
|
441
|
+
Process.should_receive(:kill).exactly(0).times
|
442
|
+
c.fork { c.die! }
|
443
|
+
end
|
444
|
+
|
445
|
+
end
|
446
|
+
|
447
|
+
context "fork_master_child_and_monitor"
|
448
|
+
context "at_exit_to_master_child"
|
449
|
+
context "at_exit_to_every_child_fork"
|
450
|
+
|
451
|
+
context "log_handler" do
|
452
|
+
|
453
|
+
it "should call worker log_handler if haven't it own" do
|
454
|
+
worker = mock("Worker")
|
455
|
+
worker.should_receive(:log_handler).and_return(123)
|
456
|
+
Beanpicker.should_receive(:log_handler).exactly(0).times
|
457
|
+
c=c("foo", {}, 0, worker)
|
458
|
+
c.log_handler.should be_equal(123)
|
459
|
+
end
|
460
|
+
|
461
|
+
it "should call Beanpicker::log_handler if haven't it own nor worker" do
|
462
|
+
Beanpicker.should_receive(:log_handler).and_return(123)
|
463
|
+
c("foo").log_handler.should be_equal(123)
|
464
|
+
end
|
465
|
+
|
466
|
+
it "should call it own log_handler if exists" do
|
467
|
+
Beanpicker.should_receive(:log_handler).exactly(0).times
|
468
|
+
c=c("foo")
|
469
|
+
c.log_handler = STDOUT
|
470
|
+
c.log_handler
|
471
|
+
end
|
472
|
+
|
473
|
+
end
|
474
|
+
|
475
|
+
end
|
476
|
+
|
477
|
+
end
|
478
|
+
|