beanpicker 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rspec +2 -0
- data/README.markdown +186 -0
- data/Rakefile +47 -0
- data/beanpicker.gemspec +61 -0
- data/bin/combine +88 -0
- data/examples/sandwich.rb +46 -0
- data/lib/beanpicker.rb +268 -0
- data/lib/beanpicker/job_server.rb +374 -0
- data/lib/beanpicker/process.rb +23 -0
- data/lib/beanpicker/version.rb +9 -0
- data/spec/beanpicker_spec.rb +478 -0
- data/spec/spec_helper.rb +168 -0
- metadata +103 -0
@@ -0,0 +1,374 @@
|
|
1
|
+
require File.expand_path( File.join(File.dirname(__FILE__), "..", "beanpicker") )
|
2
|
+
require File.expand_path( File.join(File.dirname(__FILE__), "process") )
|
3
|
+
|
4
|
+
# Used by Beanpicker to kill childs and bury jobs
|
5
|
+
BEANPICKER_FORK = { :child_every => false, :child_master => false, :child_pid => 0 }
|
6
|
+
|
7
|
+
module Beanpicker
|
8
|
+
|
9
|
+
# Used by combine to run a server of jobs
|
10
|
+
class Server
|
11
|
+
|
12
|
+
include MsgLogger
|
13
|
+
|
14
|
+
# expect a list of files to load or pick from ARGV
|
15
|
+
def initialize(args=ARGV)
|
16
|
+
@args = args
|
17
|
+
end
|
18
|
+
|
19
|
+
# Create all workers and verify if exists any Worker or Worker::Child to start a loop.
|
20
|
+
# If no only exit
|
21
|
+
def run
|
22
|
+
debug "Hiring workers..."
|
23
|
+
for arg in @args
|
24
|
+
Worker.new arg
|
25
|
+
end
|
26
|
+
|
27
|
+
w = c = 0
|
28
|
+
for worker in Beanpicker::workers
|
29
|
+
w += 1
|
30
|
+
m = "Worker: #{worker.name} hired to do"
|
31
|
+
f = {}
|
32
|
+
for child in worker.childs
|
33
|
+
c += 1
|
34
|
+
f[child.job_name] ||= 0
|
35
|
+
f[child.job_name] += 1
|
36
|
+
end
|
37
|
+
for job in f.keys.sort
|
38
|
+
m << " #{job}##{f[job]}"
|
39
|
+
end
|
40
|
+
debug m
|
41
|
+
end
|
42
|
+
|
43
|
+
if w == 0
|
44
|
+
fatal ["NOBODY WANT TO WORK TO YOU!!",
|
45
|
+
"Have you specified a file?",
|
46
|
+
"e.g. #{$0} lib/my_file_with_jobs.rb"].join("\n")
|
47
|
+
exit
|
48
|
+
end
|
49
|
+
|
50
|
+
if c == 0
|
51
|
+
fatal ["ALL YOUR #{w} WORKER#{"S" if w > 1} ARE LAZY AND DON'T WANT TO DO ANYTHING!!",
|
52
|
+
"Have you specified a job in any of yous files?",
|
53
|
+
"e.g. job('no.lazy.worker') { |args| puts args[:my_money] }"].join("\n")
|
54
|
+
exit
|
55
|
+
end
|
56
|
+
|
57
|
+
debug "#{w} worker#{"s" if w > 1} hired to do #{c} job#{"s" if c > 1}, counting the money..."
|
58
|
+
|
59
|
+
loop { sleep 1 }
|
60
|
+
end
|
61
|
+
|
62
|
+
# Use the Beanpicker::log_handler to log messages
|
63
|
+
def log_handler
|
64
|
+
Beanpicker::log_handler
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
# Worker is the class used to create jobs
|
70
|
+
class Worker
|
71
|
+
|
72
|
+
include MsgLogger
|
73
|
+
|
74
|
+
# The name of worker. Pick from args[:name] or from the name of file(if passed)
|
75
|
+
attr_reader :name
|
76
|
+
# The list of childs
|
77
|
+
attr_reader :childs
|
78
|
+
|
79
|
+
# Accept a file and/or a block with jobs and add itself to workers list
|
80
|
+
def initialize(filepath=nil, args={}, &blk)
|
81
|
+
@childs = []
|
82
|
+
@name = args[:name] rescue nil
|
83
|
+
if filepath
|
84
|
+
@name = filepath.split(/[\\\/]/)[-1].gsub(/\.[^\.]+$/,'').split(/[_\.]/).map do |x|
|
85
|
+
x.capitalize
|
86
|
+
end.join if @name.nil?
|
87
|
+
|
88
|
+
begin
|
89
|
+
instance_eval File.read(filepath)
|
90
|
+
rescue => e
|
91
|
+
error Beanpicker::exception_message(e, "when loading file #{filepath}")
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
if block_given?
|
96
|
+
begin
|
97
|
+
instance_eval(&blk)
|
98
|
+
rescue => e
|
99
|
+
error Beanpicker::exception_message(e, "when evaluating block")
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
@name = "BeanpickerWorker without name" unless @name
|
104
|
+
Beanpicker::add_worker(self)
|
105
|
+
end
|
106
|
+
|
107
|
+
# Pass the job to Child::process and add the return to childs list
|
108
|
+
def job(name, args={}, &blk)
|
109
|
+
@childs << Child.process(name, args, self, &blk)
|
110
|
+
@childs.flatten!
|
111
|
+
end
|
112
|
+
|
113
|
+
# Use it own log_handler or global if not defined
|
114
|
+
def log_handler
|
115
|
+
@log_handler || Beanpicker::log_handler
|
116
|
+
end
|
117
|
+
|
118
|
+
# Shortcut to log_handler=
|
119
|
+
def log_file(f)
|
120
|
+
#without self call don't call log_handler= of this class Oo
|
121
|
+
self.log_handler = f
|
122
|
+
end
|
123
|
+
|
124
|
+
# Child is the class that handle the job.
|
125
|
+
# Every job can be one or more childs
|
126
|
+
class Child
|
127
|
+
|
128
|
+
include MsgLogger
|
129
|
+
|
130
|
+
# Process job.
|
131
|
+
# Use opts[:childs] or Beanpicker::default_childs_number to determine how many childs should be created
|
132
|
+
def self.process(job, opts={}, worker=nil, &blk)
|
133
|
+
(opts[:childs] || Beanpicker::default_childs_number).times.map do |i|
|
134
|
+
Child.new(job, opts, i, worker, &blk)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
|
139
|
+
# The name of job
|
140
|
+
attr_reader :job_name
|
141
|
+
# The number of job(generated by Child::process)
|
142
|
+
attr_reader :number
|
143
|
+
# Should fork every job?
|
144
|
+
attr_reader :fork_every
|
145
|
+
# Should fork the child process?
|
146
|
+
attr_reader :fork_master
|
147
|
+
# The pid of running job
|
148
|
+
attr_reader :fork_every_pid
|
149
|
+
# The pid of forked child process
|
150
|
+
attr_reader :fork_master_pid
|
151
|
+
# The default options merged with argument options
|
152
|
+
attr_reader :opts
|
153
|
+
# The Worker (father?) or nil
|
154
|
+
attr_reader :worker
|
155
|
+
|
156
|
+
# Create a new job, start a fork if @opts[:fork_master] and start the work
|
157
|
+
def initialize(job, opts={}, number=0, worker=nil, &blk)
|
158
|
+
@job_name = job
|
159
|
+
@opts = {
|
160
|
+
:childs => Beanpicker::default_childs_number,
|
161
|
+
:fork_every => Beanpicker::default_fork_every,
|
162
|
+
:fork_master => Beanpicker::default_fork_master
|
163
|
+
}.merge(opts)
|
164
|
+
@number = number
|
165
|
+
@blk = blk
|
166
|
+
@loop = nil
|
167
|
+
@beanstalk = Beanpicker::new_beanstalk
|
168
|
+
@run = true
|
169
|
+
@job = nil
|
170
|
+
@worker = worker
|
171
|
+
if @opts[:fork]
|
172
|
+
_fork_every = @opts[:fork].to_s == 'every'
|
173
|
+
_fork_master = @opts[:fork].to_s == 'master'
|
174
|
+
else
|
175
|
+
_fork_every = !!@opts[:fork_every]
|
176
|
+
_fork_master = !!@opts[:fork_master]
|
177
|
+
end
|
178
|
+
#really need self
|
179
|
+
self.log_handler = @opts[:log_file] unless @opts[:log_file].nil?
|
180
|
+
@fork_every = Beanpicker::fork_every.nil? ? _fork_every : Beanpicker::fork_every
|
181
|
+
@fork_master = Beanpicker::fork_master.nil? ? _fork_master : Beanpicker::fork_master
|
182
|
+
@fork_master_pid = nil
|
183
|
+
@fork_every_pid = nil
|
184
|
+
start_watch
|
185
|
+
start_loop
|
186
|
+
end
|
187
|
+
|
188
|
+
# Return the beanstalk connection(Child don't use the global connection)
|
189
|
+
def beanstalk
|
190
|
+
@beanstalk
|
191
|
+
end
|
192
|
+
|
193
|
+
# Watch the tube with job name and ignore all others
|
194
|
+
def start_watch
|
195
|
+
beanstalk.watch(@job_name)
|
196
|
+
beanstalk.list_tubes_watched.each do |server, tubes|
|
197
|
+
tubes.each { |tube| beanstalk.ignore(tube) unless tube == @job_name }
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
# Start the loop, fork if needed
|
202
|
+
def start_loop
|
203
|
+
return false if @loop and @loop.alive?
|
204
|
+
if @fork_master
|
205
|
+
fork_master_child_and_monitor
|
206
|
+
else
|
207
|
+
@loop = Thread.new(self) do |child|
|
208
|
+
work_loop(child)
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# call start_work passing itself(child) while @run
|
214
|
+
def work_loop(child)
|
215
|
+
start_work(child) while @run
|
216
|
+
end
|
217
|
+
|
218
|
+
# Here is all the magic :)
|
219
|
+
#
|
220
|
+
# Call fork and start the job and delete on end, bury the job if a error is raised
|
221
|
+
def start_work(child=self)
|
222
|
+
fork do
|
223
|
+
begin
|
224
|
+
@job = child.beanstalk.reserve
|
225
|
+
BEANPICKER_FORK[:job] = @job if BEANPICKER_FORK[:child_every]
|
226
|
+
data = @job.ybody
|
227
|
+
|
228
|
+
if not data.is_a?(Hash) or [:args, :next_jobs] - data.keys != []
|
229
|
+
data = { :args => data, :next_jobs => [] }
|
230
|
+
end
|
231
|
+
|
232
|
+
t=Time.now
|
233
|
+
debug "Running #{@job_name}##{@number} with args #{data[:args]}; next jobs #{data[:next_jobs]}"
|
234
|
+
r = @blk.call(data[:args].clone)
|
235
|
+
debug "Job #{@job_name}##{@number} finished in #{Time.now-t} seconds with return #{r}"
|
236
|
+
data[:args].merge!(r) if r.is_a?(Hash) and data[:args].is_a?(Hash)
|
237
|
+
|
238
|
+
@job.delete
|
239
|
+
|
240
|
+
Beanpicker.enqueue(data[:next_jobs], data[:args]) if r and not data[:next_jobs].empty?
|
241
|
+
rescue => e
|
242
|
+
fatal Beanpicker::exception_message(e, "in loop of #{@job_name}##{@number} with pid #{Process.pid}")
|
243
|
+
if BEANPICKER_FORK[:child_every]
|
244
|
+
exit
|
245
|
+
else
|
246
|
+
Thread.new(@job) { |j| j.bury rescue nil }
|
247
|
+
end
|
248
|
+
ensure
|
249
|
+
@job = nil
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# Fork the process if @fork_every and wait or only call the block
|
255
|
+
def fork(&blk)
|
256
|
+
if @fork_every
|
257
|
+
@fork_every_pid = pid = Kernel.fork do
|
258
|
+
BEANPICKER_FORK[:child_every] = true
|
259
|
+
Process.die_with_parent
|
260
|
+
at_exit_to_every_child_fork
|
261
|
+
$0 = "Beanpicker job child #{@job_name}##{@number} of #{Process.ppid}"
|
262
|
+
blk.call
|
263
|
+
end
|
264
|
+
if BEANPICKER_FORK[:child_master]
|
265
|
+
BEANPICKER_FORK[:child_pid] = pid
|
266
|
+
Process.waitpid pid
|
267
|
+
BEANPICKER_FORK[:child_pid] = nil
|
268
|
+
else
|
269
|
+
Process.waitpid pid
|
270
|
+
end
|
271
|
+
@fork_every_pid = nil
|
272
|
+
else
|
273
|
+
blk.call
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
# The child still working?
|
278
|
+
def running?
|
279
|
+
@run
|
280
|
+
end
|
281
|
+
|
282
|
+
# Stop running, kill the thread and kill master os every process
|
283
|
+
def die!
|
284
|
+
@run = false
|
285
|
+
@loop.kill if @loop and @loop.alive?
|
286
|
+
|
287
|
+
kill_pid = nil
|
288
|
+
if @fork_master
|
289
|
+
kill_pid = @fork_master_pid
|
290
|
+
elsif @fork_every
|
291
|
+
kill_pid = @fork_every_pid
|
292
|
+
end
|
293
|
+
|
294
|
+
if kill_pid and kill_pid.is_a?(Integer) and Process.running?(kill_pid)
|
295
|
+
debug "Killing child with pid #{kill_pid}"
|
296
|
+
Process.kill "TERM", kill_pid
|
297
|
+
end
|
298
|
+
|
299
|
+
end
|
300
|
+
|
301
|
+
# Crete a new fork, change the name and create a thread to restart the fork if it die
|
302
|
+
#
|
303
|
+
# Called by start_loop when fork_master
|
304
|
+
def fork_master_child_and_monitor
|
305
|
+
@fork_master_pid = Kernel.fork do
|
306
|
+
at_exit_to_master_child_fork
|
307
|
+
Process.die_with_parent
|
308
|
+
BEANPICKER_FORK[:child_master] = true
|
309
|
+
$0 = "Beanpicker master child #{@job_name}##{@number}"
|
310
|
+
work_loop(self)
|
311
|
+
end
|
312
|
+
@loop = Thread.new(self) do |child|
|
313
|
+
Process.waitpid @fork_master_pid
|
314
|
+
child.fork_master_child_and_monitor if child.running?
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
318
|
+
# Create a at_exit to kill the child(if any)
|
319
|
+
#
|
320
|
+
# Used by fork_master_child_and_monitor
|
321
|
+
def at_exit_to_master_child_fork
|
322
|
+
at_exit do
|
323
|
+
pid = BEANPICKER_FORK[:child_pid]
|
324
|
+
if pid and pid > 0
|
325
|
+
if Process.running?(pid)
|
326
|
+
Process.kill "TERM", pid
|
327
|
+
sleep 0.1
|
328
|
+
if Process.running?(pid)
|
329
|
+
sleep 2
|
330
|
+
Process.kill "KILL", pid if Process.running?(pid)
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
334
|
+
Kernel.exit!
|
335
|
+
end
|
336
|
+
end
|
337
|
+
|
338
|
+
# Create a at_exit to bury the job(if any) and exit
|
339
|
+
#
|
340
|
+
# Used by fork
|
341
|
+
def at_exit_to_every_child_fork
|
342
|
+
at_exit do
|
343
|
+
Thread.new do
|
344
|
+
sleep 1
|
345
|
+
Kernel.exit!
|
346
|
+
end
|
347
|
+
BEANPICKER_FORK[:job].bury rescue nil if BEANPICKER_FORK[:job]
|
348
|
+
end
|
349
|
+
end
|
350
|
+
|
351
|
+
# Return the own log_handler, or the Worker log_handler or the global log_handler
|
352
|
+
def log_handler
|
353
|
+
#'@log_handler || ' go to worker/global log_handler even if @log_handler is defined
|
354
|
+
defined?(@log_handler) ? @log_handler : @worker.nil? ? Beanpicker::log_handler : @worker.log_handler
|
355
|
+
end
|
356
|
+
|
357
|
+
end
|
358
|
+
|
359
|
+
end
|
360
|
+
|
361
|
+
end
|
362
|
+
|
363
|
+
# kill childs or stop jobs
|
364
|
+
at_exit do
|
365
|
+
if not BEANPICKER_FORK[:child_master] and not BEANPICKER_FORK[:child_every]
|
366
|
+
Beanpicker::debug "Laying off workers..." if Beanpicker::workers.count > 0
|
367
|
+
Beanpicker::stop_workers
|
368
|
+
end
|
369
|
+
end
|
370
|
+
|
371
|
+
# hide errors throwed with ctrl+c
|
372
|
+
trap("INT") {
|
373
|
+
exit
|
374
|
+
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Process Premium Version + Crack + Serial + two new methods
|
2
|
+
module Process
|
3
|
+
|
4
|
+
# Verify if a process is running
|
5
|
+
def self.running?(pid)
|
6
|
+
begin
|
7
|
+
Process.kill 0, pid
|
8
|
+
return true
|
9
|
+
rescue Errno::ESRCH, Errno::EPERM
|
10
|
+
return false
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Create a thread to kill the current process if the parent process die
|
15
|
+
def self.die_with_parent
|
16
|
+
Thread.new do
|
17
|
+
while Process.running?(Process.ppid)
|
18
|
+
sleep 1
|
19
|
+
end
|
20
|
+
Kernel.exit
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,478 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Beanpicker do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
Beanpicker::default_fork_every = false
|
7
|
+
Beanpicker::default_fork_master = false
|
8
|
+
end
|
9
|
+
|
10
|
+
before(:each) do
|
11
|
+
@beanstalk = Beanpicker::beanstalk
|
12
|
+
@ms = { :values => {}, :methods => [:default_fork_every,
|
13
|
+
:default_fork_master,
|
14
|
+
:fork_every,
|
15
|
+
:fork_master,
|
16
|
+
:default_childs_number] }
|
17
|
+
for m in @ms[:methods]
|
18
|
+
@ms[:values][m] = Beanpicker.send(m)
|
19
|
+
end
|
20
|
+
|
21
|
+
Thread.stub!(:new)
|
22
|
+
Kernel.stub!(:fork)
|
23
|
+
Process.stub!(:waitpid)
|
24
|
+
end
|
25
|
+
|
26
|
+
after(:each) do
|
27
|
+
for m in @ms[:methods]
|
28
|
+
Beanpicker.send("#{m}=", @ms[:values][m])
|
29
|
+
end
|
30
|
+
Beanpicker::workers.replace []
|
31
|
+
end
|
32
|
+
|
33
|
+
context "Getters and Setters" do
|
34
|
+
|
35
|
+
describe "log_handler" do
|
36
|
+
|
37
|
+
it "should use STDOUT by default" do
|
38
|
+
Logger.should_receive(:new).with(STDOUT).once
|
39
|
+
Beanpicker::log_handler
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should use string as a argument to Logger" do
|
43
|
+
Logger.should_receive(:new).with("/dev/null").once
|
44
|
+
Beanpicker::log_handler = "/dev/null"
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should use IO as a argument to Logger" do
|
48
|
+
Logger.should_receive(:new).with(STDERR).once
|
49
|
+
Beanpicker::log_handler = STDERR
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should use anything that isn't IO/String and respond to 5 log methods as a logger" do
|
53
|
+
l=Logger.new(STDOUT)
|
54
|
+
Beanpicker::log_handler = l
|
55
|
+
Beanpicker::log_handler.should be_equal(l)
|
56
|
+
end
|
57
|
+
|
58
|
+
it "should not use anything that ins't IO/String and don't respond to 5 log methods as a logger" do
|
59
|
+
Beanpicker.should_receive(:error).once
|
60
|
+
l = Beanpicker::log_handler
|
61
|
+
Beanpicker::log_handler = :foo
|
62
|
+
Beanpicker::log_handler.should be_equal(l)
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
it "default_pri" do
|
68
|
+
Beanpicker::default_pri = 10
|
69
|
+
Beanpicker::default_pri.should be_equal(10)
|
70
|
+
end
|
71
|
+
|
72
|
+
it "default_delay" do
|
73
|
+
Beanpicker::default_delay = 10
|
74
|
+
Beanpicker::default_delay.should be_equal(10)
|
75
|
+
end
|
76
|
+
|
77
|
+
it "default_ttr" do
|
78
|
+
Beanpicker::default_ttr = 10
|
79
|
+
Beanpicker::default_ttr.should be_equal(10)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "default_childs_number" do
|
83
|
+
Beanpicker::default_childs_number = 10
|
84
|
+
Beanpicker::default_childs_number.should be_equal(10)
|
85
|
+
end
|
86
|
+
|
87
|
+
it "default_fork_every" do
|
88
|
+
Beanpicker::default_fork_every = true
|
89
|
+
Beanpicker::default_fork_every.should be_true
|
90
|
+
Beanpicker::default_fork_every = false
|
91
|
+
Beanpicker::default_fork_every.should be_false
|
92
|
+
end
|
93
|
+
|
94
|
+
it "default_fork_master" do
|
95
|
+
Beanpicker::default_fork_master = true
|
96
|
+
Beanpicker::default_fork_master.should be_true
|
97
|
+
Beanpicker::default_fork_master = false
|
98
|
+
Beanpicker::default_fork_master.should be_false
|
99
|
+
end
|
100
|
+
|
101
|
+
it "fork_every" do
|
102
|
+
Beanpicker::fork_every = true
|
103
|
+
Beanpicker::fork_every.should be_true
|
104
|
+
Beanpicker::fork_every = false
|
105
|
+
Beanpicker::fork_every.should be_false
|
106
|
+
end
|
107
|
+
|
108
|
+
it "fork_master" do
|
109
|
+
Beanpicker::fork_master = true
|
110
|
+
Beanpicker::fork_master.should be_true
|
111
|
+
Beanpicker::fork_master = false
|
112
|
+
Beanpicker::fork_master.should be_false
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
117
|
+
describe "enqueue" do
|
118
|
+
|
119
|
+
it "should use job name as tube and send via yput" do
|
120
|
+
@beanstalk.should_receive(:use).once.with("foo.bar")
|
121
|
+
@beanstalk.should_receive(:yput).once.with({ :args => { :foo => :bar }, :next_jobs => [] }, 1, 2, 3)
|
122
|
+
Beanpicker.enqueue("foo.bar", { :foo => :bar }, { :pri => 1, :delay => 2, :ttr => 3 })
|
123
|
+
end
|
124
|
+
|
125
|
+
it "should use default pri, delay and ttr and argument as a empty hash" do
|
126
|
+
@beanstalk.should_receive(:yput).once.with({ :args => {}, :next_jobs => [] },
|
127
|
+
Beanpicker::default_pri,
|
128
|
+
Beanpicker::default_delay,
|
129
|
+
Beanpicker::default_ttr)
|
130
|
+
Beanpicker::enqueue("foo.bar")
|
131
|
+
end
|
132
|
+
|
133
|
+
it "should understand chain jobs" do
|
134
|
+
@beanstalk.should_receive(:yput).once.with({ :args => {}, :next_jobs => ["foo.bar2", "foo.bar3"]}, 1, 2, 3)
|
135
|
+
@beanstalk.should_receive(:use).once.with("foo.bar")
|
136
|
+
Beanpicker::enqueue(["foo.bar", "foo.bar2", "foo.bar3"], {},
|
137
|
+
{ :pri => 1, :delay => 2, :ttr => 3 })
|
138
|
+
end
|
139
|
+
|
140
|
+
it "should handle connection error and retry" do
|
141
|
+
Beanpicker::beanstalk.should be_equal(@beanstalk)
|
142
|
+
@beanstalk.close
|
143
|
+
b = Beanpicker::new_beanstalk
|
144
|
+
TCPSocket.fail_times(1)
|
145
|
+
Beanpicker.should_receive(:new_beanstalk).and_return(b)
|
146
|
+
Beanpicker.should_receive(:exception_message).once.with(an_instance_of(Beanstalk::NotConnected), an_instance_of(String))
|
147
|
+
Beanpicker.should_receive(:error).once
|
148
|
+
Beanpicker.enqueue("foo.bar")
|
149
|
+
end
|
150
|
+
|
151
|
+
it "should raise a exeption if a retry don't work" do
|
152
|
+
@beanstalk.close
|
153
|
+
TCPSocket.fail_times 2
|
154
|
+
Beanpicker.should_receive(:error).once
|
155
|
+
expect { Beanpicker::enqueue("foo.bar") }.should raise_error(Beanstalk::NotConnected)
|
156
|
+
TCPSocket.fail_times 0
|
157
|
+
end
|
158
|
+
|
159
|
+
|
160
|
+
end
|
161
|
+
|
162
|
+
context "new beanstalk" do
|
163
|
+
|
164
|
+
it "new_beanstalk should create a new instance of Beanstalk::Pool every time" do
|
165
|
+
Beanpicker::new_beanstalk.should_not be_equal(Beanpicker::new_beanstalk)
|
166
|
+
end
|
167
|
+
|
168
|
+
it "new_beanstalk_should use beanstalk_urls and return [localhost:11300] by default" do
|
169
|
+
Beanpicker::beanstalk_urls.should include("localhost:11300")
|
170
|
+
Beanpicker.should_receive(:beanstalk_urls).and_return(["localhost:11300"])
|
171
|
+
Beanpicker::new_beanstalk
|
172
|
+
end
|
173
|
+
|
174
|
+
it "beanstalk_urls should read ENV['BEANSTALK_URL'] and ENV['BEANSTALK_URLS']" do
|
175
|
+
ENV.should_receive(:[]).with("BEANSTALK_URL").and_return("localhost:3000")
|
176
|
+
ENV.should_receive(:[]).with("BEANSTALK_URLS").and_return("localhost:1500,beanstalk://www.foo.bar.net:4500")
|
177
|
+
urls = Beanpicker::beanstalk_urls
|
178
|
+
urls.should include("localhost:3000")
|
179
|
+
urls.should include("localhost:1500")
|
180
|
+
urls.should include("www.foo.bar.net:4500")
|
181
|
+
urls.should_not include("localhost:11300")
|
182
|
+
end
|
183
|
+
|
184
|
+
end
|
185
|
+
|
186
|
+
it "add_worker should add a worker(duh!)" do
|
187
|
+
expect { Beanpicker::add_worker 1 }.should change(Beanpicker::workers, :size).by(1)
|
188
|
+
end
|
189
|
+
|
190
|
+
it "stop_workers should kill childs of workers" do
|
191
|
+
child = mock(Beanpicker::Worker::Child)
|
192
|
+
worker = mock(Beanpicker::Worker)
|
193
|
+
worker.should_receive(:childs).and_return([child])
|
194
|
+
child.should_receive(:die!)
|
195
|
+
Beanpicker::workers.should_receive(:clear)
|
196
|
+
Beanpicker::add_worker worker
|
197
|
+
Beanpicker::stop_workers
|
198
|
+
end
|
199
|
+
|
200
|
+
describe Beanpicker::Worker do
|
201
|
+
|
202
|
+
it "should handle a error when loading a file" do
|
203
|
+
pending "don't know how make"
|
204
|
+
Beanpicker::Worker.should_receive(:error)
|
205
|
+
Beanpicker::Worker.new("/foo/bar/lol.rb")
|
206
|
+
end
|
207
|
+
|
208
|
+
it "should handle a error when evaluating a block" do
|
209
|
+
pending "don't know how make"
|
210
|
+
Beanpicker::Worker.new do
|
211
|
+
foo
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
it "should add itself to workers list" do
|
216
|
+
expect { Beanpicker::Worker.new }.should change(Beanpicker::workers, :count).by(1)
|
217
|
+
Beanpicker::workers.should include(Beanpicker::Worker.new)
|
218
|
+
end
|
219
|
+
|
220
|
+
it "job should call Child.process" do
|
221
|
+
w=Beanpicker::Worker.new
|
222
|
+
Beanpicker::Worker::Child.should_receive(:process).and_return([1])
|
223
|
+
expect { w.job("foo.bar") }.should change(w.childs, :count).by(1)
|
224
|
+
end
|
225
|
+
|
226
|
+
context "log_handler" do
|
227
|
+
|
228
|
+
it "should use global log_handler if haven't it own defined" do
|
229
|
+
l = Logger.new(STDOUT)
|
230
|
+
Beanpicker.should_receive(:log_handler).twice.and_return(l)
|
231
|
+
l.should_receive(:debug)
|
232
|
+
w = Beanpicker::Worker.new
|
233
|
+
w.log_handler.should be_equal(l)
|
234
|
+
w.debug("foo!")
|
235
|
+
end
|
236
|
+
|
237
|
+
it "should use it own log_handler if defined" do
|
238
|
+
w=Beanpicker::Worker.new
|
239
|
+
w.should_receive(:debug)
|
240
|
+
w.log_handler = STDOUT
|
241
|
+
w.debug("foo!")
|
242
|
+
end
|
243
|
+
|
244
|
+
it "should use log_file as a mirror to log_handler" do
|
245
|
+
w=Beanpicker::Worker.new
|
246
|
+
w.should_receive(:log_handler=)
|
247
|
+
w.log_file 'foo'
|
248
|
+
end
|
249
|
+
|
250
|
+
end
|
251
|
+
|
252
|
+
end
|
253
|
+
|
254
|
+
describe Beanpicker::Worker::Child do
|
255
|
+
|
256
|
+
|
257
|
+
def c(*a, &blk)
|
258
|
+
Beanpicker::Worker::Child.new(*a, &blk)
|
259
|
+
end
|
260
|
+
|
261
|
+
context "Child.process"do
|
262
|
+
|
263
|
+
it "should create one child by default" do
|
264
|
+
Beanpicker::Worker::Child.process("foo").count.should be_equal(1)
|
265
|
+
end
|
266
|
+
|
267
|
+
it "should respect default_childs_number if no option :childs is passed" do
|
268
|
+
Beanpicker::default_childs_number = 3
|
269
|
+
Beanpicker::Worker::Child.process("foo").count.should be_equal(3)
|
270
|
+
end
|
271
|
+
|
272
|
+
it "should respect :childs if passed" do
|
273
|
+
Beanpicker::Worker::Child.process("foo", { :childs => 3 }).count.should be_equal(3)
|
274
|
+
end
|
275
|
+
|
276
|
+
end
|
277
|
+
|
278
|
+
context "new" do
|
279
|
+
|
280
|
+
it "should create it own beanstalk instance" do
|
281
|
+
b = Beanpicker::new_beanstalk
|
282
|
+
Beanpicker.should_receive(:new_beanstalk).and_return(b)
|
283
|
+
c("foo", {}, 1)
|
284
|
+
end
|
285
|
+
|
286
|
+
it "should define job_name, number and opts and worker" do
|
287
|
+
j = "foo"
|
288
|
+
n = 1
|
289
|
+
o = { :foo => :bar }
|
290
|
+
w = mock("Worker")
|
291
|
+
child = c(j, o, n, w)
|
292
|
+
child.job_name.should be_equal(j)
|
293
|
+
child.number.should be_equal(n)
|
294
|
+
child.opts.should_not be_equal(o)
|
295
|
+
child.opts[:foo].should be_equal(o[:foo])
|
296
|
+
child.worker.should be_equal(w)
|
297
|
+
end
|
298
|
+
|
299
|
+
it "should understand :fork_every and :fork_master" do
|
300
|
+
c = c("foo", { :fork_every => true, :fork_master => true })
|
301
|
+
c.fork_every.should be_true
|
302
|
+
c.fork_master.should be_true
|
303
|
+
end
|
304
|
+
|
305
|
+
it "should overwrite :fork_every and :fork_master with :fork" do
|
306
|
+
c1 = c("foo", { :fork_every => false, :fork_master => true, :fork => :every })
|
307
|
+
c2 = c("foo", { :fork_every => true, :fork_master => false, :fork => :master })
|
308
|
+
c1.fork_every.should be_true
|
309
|
+
c1.fork_master.should be_false
|
310
|
+
c2.fork_every.should be_false
|
311
|
+
c2.fork_master.should be_true
|
312
|
+
end
|
313
|
+
|
314
|
+
it "should create it own log_handler if :log_file is passed and is a String or a IO" do
|
315
|
+
l = Logger.new STDOUT
|
316
|
+
Beanpicker.should_receive(:log_handler).exactly(0).times
|
317
|
+
c1=c("foo", { :log_file => l })
|
318
|
+
c2=c("foo", { :log_file => "/dev/null" })
|
319
|
+
c1.log_handler.should be_equal(l)
|
320
|
+
c2.log_handler
|
321
|
+
end
|
322
|
+
|
323
|
+
end
|
324
|
+
|
325
|
+
it "should watch only the tube with job_name" do
|
326
|
+
c = c("foo")
|
327
|
+
b = c.beanstalk
|
328
|
+
b.watch("bar")
|
329
|
+
b.should_receive(:watch).with("foo")
|
330
|
+
b.should_receive(:ignore).with("bar")
|
331
|
+
c.start_watch
|
332
|
+
end
|
333
|
+
|
334
|
+
context "start_loop" do
|
335
|
+
|
336
|
+
it "should call fork_master_child_and_monitor if have fork_master" do
|
337
|
+
c=c("foo", :fork_master => true)
|
338
|
+
c.should_receive(:fork_master_child_and_monitor)
|
339
|
+
c.start_loop
|
340
|
+
end
|
341
|
+
|
342
|
+
it "should call Thread.new if haven't fork_master" do
|
343
|
+
Thread.should_receive(:new)
|
344
|
+
c=c("foo", :fork_master => false)
|
345
|
+
end
|
346
|
+
|
347
|
+
end
|
348
|
+
|
349
|
+
context "fork" do
|
350
|
+
|
351
|
+
it "should call Kernel.fork if have fork_every" do
|
352
|
+
c=c("foo", :fork_every => true)
|
353
|
+
Kernel.should_receive(:fork).once.and_return(0)
|
354
|
+
c.fork {}
|
355
|
+
end
|
356
|
+
|
357
|
+
it "should call only block if haven't fork_every" do
|
358
|
+
c=c("foo", :fork_every => false)
|
359
|
+
b = proc {}
|
360
|
+
Kernel.should_receive(:fork).exactly(0).times
|
361
|
+
b.should_receive(:call).once
|
362
|
+
c.fork(&b)
|
363
|
+
end
|
364
|
+
|
365
|
+
end
|
366
|
+
|
367
|
+
context "start_work" do
|
368
|
+
|
369
|
+
before(:each) do
|
370
|
+
@beanstalk.yput({ :args => { :foo => :bar }})
|
371
|
+
@job = @beanstalk.reserve
|
372
|
+
end
|
373
|
+
|
374
|
+
it "should reserve and delete a job" do
|
375
|
+
c=c("foo"){}
|
376
|
+
c.beanstalk.should_receive(:reserve).once.and_return(@job)
|
377
|
+
@job.should_receive(:ybody).once.and_return({})
|
378
|
+
@job.should_receive(:delete).once
|
379
|
+
c.should_receive(:fatal).exactly(0).times
|
380
|
+
c.start_work
|
381
|
+
end
|
382
|
+
|
383
|
+
it "should bury the job if got a eror" do
|
384
|
+
c=c("foo"){ raise RuntimeError }
|
385
|
+
@job.should_receive(:delete).exactly(0).times
|
386
|
+
@job.should_receive(:bury).once
|
387
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
388
|
+
Thread.should_receive(:new).once.with(@job).and_yield(@job)
|
389
|
+
c.start_work
|
390
|
+
end
|
391
|
+
|
392
|
+
it "should not do chain jobs if receive false from block" do
|
393
|
+
c=c("foo"){ false }
|
394
|
+
@job.ybody[:next_jobs] = ["foo"]
|
395
|
+
Beanpicker.should_receive(:enqueue).exactly(0).times
|
396
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
397
|
+
c.start_work
|
398
|
+
end
|
399
|
+
|
400
|
+
it "should not do chain jobs if receive nil from block" do
|
401
|
+
c=c("foo"){ nil }
|
402
|
+
@job.ybody[:next_jobs] = ["foo"]
|
403
|
+
Beanpicker.should_receive(:enqueue).exactly(0).times
|
404
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
405
|
+
c.start_work
|
406
|
+
end
|
407
|
+
|
408
|
+
it "should do chain jobs if receive a positive return from block" do
|
409
|
+
c=c("foo"){ true }
|
410
|
+
@job.ybody[:next_jobs] = ["foo"]
|
411
|
+
Beanpicker.should_receive(:enqueue).once.with(@job.ybody[:next_jobs], @job.ybody[:args])
|
412
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
413
|
+
c.start_work
|
414
|
+
end
|
415
|
+
|
416
|
+
it "should do chain jobs with merged args if receive a hash from block" do
|
417
|
+
c=c("foo"){ { :lol => :bar } }
|
418
|
+
@job.ybody[:next_jobs] = ["foo"]
|
419
|
+
Beanpicker.should_receive(:enqueue).once.with(@job.ybody[:next_jobs], hash_including(@job.ybody[:args].merge({ :lol => :bar })))
|
420
|
+
c.beanstalk.should_receive(:reserve).and_return(@job)
|
421
|
+
c.start_work
|
422
|
+
end
|
423
|
+
|
424
|
+
end
|
425
|
+
|
426
|
+
context "die!" do
|
427
|
+
|
428
|
+
it "should call Process.kill if have a forked child" do
|
429
|
+
pending "with die! call it two times Oo"
|
430
|
+
c=c("foo", { :fork_every => true }){}
|
431
|
+
Kernel.should_receive(:fork).once.and_return(1234)
|
432
|
+
Process.should_receive(:waitpid).once.with { c.die!; 1234 }
|
433
|
+
Process.should_receive(:running?).once.with(1234).and_return(true)
|
434
|
+
Process.should_receive(:kill).once.with("TERM", 1234)
|
435
|
+
c.fork
|
436
|
+
end
|
437
|
+
|
438
|
+
it "should not call Process.kill if haven't a forked child" do
|
439
|
+
c=c("foo", { :fork_every => false }){}
|
440
|
+
Process.should_receive(:running?).exactly(0).times
|
441
|
+
Process.should_receive(:kill).exactly(0).times
|
442
|
+
c.fork { c.die! }
|
443
|
+
end
|
444
|
+
|
445
|
+
end
|
446
|
+
|
447
|
+
context "fork_master_child_and_monitor"
|
448
|
+
context "at_exit_to_master_child"
|
449
|
+
context "at_exit_to_every_child_fork"
|
450
|
+
|
451
|
+
context "log_handler" do
|
452
|
+
|
453
|
+
it "should call worker log_handler if haven't it own" do
|
454
|
+
worker = mock("Worker")
|
455
|
+
worker.should_receive(:log_handler).and_return(123)
|
456
|
+
Beanpicker.should_receive(:log_handler).exactly(0).times
|
457
|
+
c=c("foo", {}, 0, worker)
|
458
|
+
c.log_handler.should be_equal(123)
|
459
|
+
end
|
460
|
+
|
461
|
+
it "should call Beanpicker::log_handler if haven't it own nor worker" do
|
462
|
+
Beanpicker.should_receive(:log_handler).and_return(123)
|
463
|
+
c("foo").log_handler.should be_equal(123)
|
464
|
+
end
|
465
|
+
|
466
|
+
it "should call it own log_handler if exists" do
|
467
|
+
Beanpicker.should_receive(:log_handler).exactly(0).times
|
468
|
+
c=c("foo")
|
469
|
+
c.log_handler = STDOUT
|
470
|
+
c.log_handler
|
471
|
+
end
|
472
|
+
|
473
|
+
end
|
474
|
+
|
475
|
+
end
|
476
|
+
|
477
|
+
end
|
478
|
+
|