perfectqueue 0.7.32 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +6 -0
- data/ChangeLog +0 -62
- data/Gemfile +3 -0
- data/README.md +239 -0
- data/Rakefile +19 -0
- data/lib/perfectqueue.rb +68 -4
- data/lib/perfectqueue/application.rb +30 -0
- data/lib/perfectqueue/application/base.rb +27 -0
- data/lib/perfectqueue/application/dispatch.rb +73 -0
- data/lib/perfectqueue/application/router.rb +69 -0
- data/lib/perfectqueue/backend.rb +44 -47
- data/lib/perfectqueue/backend/rdb_compat.rb +298 -0
- data/lib/perfectqueue/blocking_flag.rb +84 -0
- data/lib/perfectqueue/client.rb +117 -0
- data/lib/perfectqueue/command/perfectqueue.rb +108 -323
- data/lib/perfectqueue/daemons_logger.rb +80 -0
- data/lib/perfectqueue/engine.rb +85 -123
- data/lib/perfectqueue/error.rb +53 -0
- data/lib/perfectqueue/model.rb +37 -0
- data/lib/perfectqueue/multiprocess.rb +31 -0
- data/lib/perfectqueue/multiprocess/child_process.rb +108 -0
- data/lib/perfectqueue/multiprocess/child_process_monitor.rb +109 -0
- data/lib/perfectqueue/multiprocess/fork_processor.rb +164 -0
- data/lib/perfectqueue/multiprocess/thread_processor.rb +123 -0
- data/lib/perfectqueue/queue.rb +58 -0
- data/lib/perfectqueue/runner.rb +39 -0
- data/lib/perfectqueue/signal_queue.rb +112 -0
- data/lib/perfectqueue/task.rb +103 -0
- data/lib/perfectqueue/task_metadata.rb +98 -0
- data/lib/perfectqueue/task_monitor.rb +189 -0
- data/lib/perfectqueue/task_status.rb +27 -0
- data/lib/perfectqueue/version.rb +1 -3
- data/lib/perfectqueue/worker.rb +114 -196
- data/perfectqueue.gemspec +24 -0
- data/spec/queue_spec.rb +234 -0
- data/spec/spec_helper.rb +44 -0
- data/spec/worker_spec.rb +81 -0
- metadata +93 -40
- checksums.yaml +0 -7
- data/README.rdoc +0 -224
- data/lib/perfectqueue/backend/null.rb +0 -33
- data/lib/perfectqueue/backend/rdb.rb +0 -181
- data/lib/perfectqueue/backend/simpledb.rb +0 -139
- data/test/backend_test.rb +0 -259
- data/test/cat.sh +0 -2
- data/test/echo.sh +0 -4
- data/test/exec_test.rb +0 -61
- data/test/fail.sh +0 -2
- data/test/huge.sh +0 -2
- data/test/stress.rb +0 -99
- data/test/success.sh +0 -2
- data/test/test_helper.rb +0 -19
data/test/cat.sh
DELETED
data/test/echo.sh
DELETED
data/test/exec_test.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
require File.dirname(__FILE__)+'/test_helper'
|
2
|
-
|
3
|
-
class ExecTest < Test::Unit::TestCase
|
4
|
-
it 'success' do
|
5
|
-
success_sh = File.expand_path File.dirname(__FILE__)+"/success.sh"
|
6
|
-
|
7
|
-
task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
|
8
|
-
e = PerfectQueue::ExecRunner.new(success_sh, task)
|
9
|
-
|
10
|
-
assert_nothing_raised do
|
11
|
-
e.run
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
it 'fail' do
|
16
|
-
fail_sh = File.expand_path File.dirname(__FILE__)+"/fail.sh"
|
17
|
-
|
18
|
-
task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
|
19
|
-
e = PerfectQueue::ExecRunner.new(fail_sh, task)
|
20
|
-
|
21
|
-
assert_raise(RuntimeError) do
|
22
|
-
e.run
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
it 'stdin' do
|
27
|
-
cat_sh = File.expand_path File.dirname(__FILE__)+"/cat.sh"
|
28
|
-
out_tmp = File.expand_path File.dirname(__FILE__)+"/cat.sh.tmp"
|
29
|
-
|
30
|
-
task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
|
31
|
-
e = PerfectQueue::ExecRunner.new("#{cat_sh} #{out_tmp}", task)
|
32
|
-
|
33
|
-
e.run
|
34
|
-
|
35
|
-
assert_equal 'data1', File.read(out_tmp)
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'echo' do
|
39
|
-
echo_sh = File.expand_path File.dirname(__FILE__)+"/echo.sh"
|
40
|
-
out_tmp = File.expand_path File.dirname(__FILE__)+"/echo.sh.tmp"
|
41
|
-
|
42
|
-
task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
|
43
|
-
e = PerfectQueue::ExecRunner.new("#{echo_sh} #{out_tmp}", task)
|
44
|
-
|
45
|
-
e.run
|
46
|
-
|
47
|
-
assert_equal "test1\n", File.read(out_tmp)
|
48
|
-
end
|
49
|
-
|
50
|
-
it 'huge' do
|
51
|
-
huge_sh = File.expand_path File.dirname(__FILE__)+"/huge.sh"
|
52
|
-
|
53
|
-
task = PerfectQueue::Task.new('test1', Time.now.to_i, 'data1')
|
54
|
-
e = PerfectQueue::ExecRunner.new(huge_sh, task)
|
55
|
-
|
56
|
-
e.run
|
57
|
-
|
58
|
-
# should finish
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
data/test/fail.sh
DELETED
data/test/huge.sh
DELETED
data/test/stress.rb
DELETED
@@ -1,99 +0,0 @@
|
|
1
|
-
$LOAD_PATH << File.expand_path(File.dirname(__FILE__)+"/../lib")
|
2
|
-
require 'perfectqueue'
|
3
|
-
require 'perfectqueue/backend/rdb'
|
4
|
-
require 'perfectqueue/backend/simpledb'
|
5
|
-
|
6
|
-
class StressTest
|
7
|
-
def initialize(uri, table, npt, thread)
|
8
|
-
@db_proc = Proc.new do
|
9
|
-
PerfectQueue::RDBBackend.new(uri, table)
|
10
|
-
end
|
11
|
-
@db_proc.call.create_tables
|
12
|
-
@npt = npt
|
13
|
-
@thread = thread
|
14
|
-
end
|
15
|
-
|
16
|
-
class ThreadMain < Thread
|
17
|
-
def initialize(key_prefix, db, num)
|
18
|
-
@key_prefix = key_prefix
|
19
|
-
@db = db
|
20
|
-
@num = num
|
21
|
-
super(&method(:run))
|
22
|
-
end
|
23
|
-
|
24
|
-
def run
|
25
|
-
@num.times {|i|
|
26
|
-
now = Time.now.to_i
|
27
|
-
@db.submit("#{@key_prefix}-#{i}", "data", now)
|
28
|
-
token, task = @db.acquire(now+60, now)
|
29
|
-
if token == nil
|
30
|
-
puts "acquire failed"
|
31
|
-
next
|
32
|
-
end
|
33
|
-
@db.update(token, now+70)
|
34
|
-
@db.finish(token, now+80)
|
35
|
-
}
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def run
|
40
|
-
threads = []
|
41
|
-
key_prefix = "stress-#{'%08x'%rand(2**32)}"
|
42
|
-
now = Time.now
|
43
|
-
@thread.times {|i|
|
44
|
-
threads << ThreadMain.new("#{key_prefix}-#{i}", @db_proc.call, @npt)
|
45
|
-
}
|
46
|
-
threads.each {|t|
|
47
|
-
t.join
|
48
|
-
}
|
49
|
-
finish = Time.now
|
50
|
-
|
51
|
-
elapsed = finish - now
|
52
|
-
puts "#{elapsed} sec."
|
53
|
-
puts "#{@npt * @thread / elapsed} req/sec."
|
54
|
-
puts "#{elapsed / (@npt * @thread)} sec/req."
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
require 'optparse'
|
59
|
-
|
60
|
-
op = OptionParser.new
|
61
|
-
op.banner += " <uri> <table>"
|
62
|
-
|
63
|
-
num = 100
|
64
|
-
thread = 1
|
65
|
-
|
66
|
-
op.on('-n', '--num N', Integer) {|n|
|
67
|
-
num = n
|
68
|
-
}
|
69
|
-
op.on('-t', '--thread N', Integer) {|n|
|
70
|
-
thread = n
|
71
|
-
}
|
72
|
-
|
73
|
-
begin
|
74
|
-
op.parse!(ARGV)
|
75
|
-
|
76
|
-
if ARGV.length != 2
|
77
|
-
puts op.to_s
|
78
|
-
exit 1
|
79
|
-
end
|
80
|
-
|
81
|
-
uri = ARGV[0]
|
82
|
-
table = ARGV[1]
|
83
|
-
|
84
|
-
rescue
|
85
|
-
puts op.to_s
|
86
|
-
puts $!
|
87
|
-
exit 1
|
88
|
-
end
|
89
|
-
|
90
|
-
npt = num / thread
|
91
|
-
num = npt * thread
|
92
|
-
|
93
|
-
puts "num: #{num}"
|
94
|
-
puts "thread: #{thread}"
|
95
|
-
puts "num/thread: #{npt}"
|
96
|
-
|
97
|
-
t = StressTest.new(uri, table, npt, thread)
|
98
|
-
t.run
|
99
|
-
|
data/test/success.sh
DELETED
data/test/test_helper.rb
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
require 'simplecov'
|
2
|
-
SimpleCov.start
|
3
|
-
require 'test/unit'
|
4
|
-
$LOAD_PATH << File.dirname(__FILE__)+"/../lib"
|
5
|
-
require 'perfectqueue'
|
6
|
-
require 'shellwords'
|
7
|
-
require 'perfectqueue/backend/rdb'
|
8
|
-
require 'perfectqueue/backend/simpledb'
|
9
|
-
require 'fileutils'
|
10
|
-
|
11
|
-
class Test::Unit::TestCase
|
12
|
-
#class << self
|
13
|
-
# alias_method :it, :test
|
14
|
-
#end
|
15
|
-
def self.it(name, &block)
|
16
|
-
define_method("test_#{name}", &block)
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|