rainbows 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/GIT-VERSION-GEN +1 -1
- data/TODO +15 -0
- data/TUNING +14 -0
- data/lib/rainbows.rb +7 -0
- data/lib/rainbows/app_pool.rb +4 -3
- data/lib/rainbows/base.rb +17 -11
- data/lib/rainbows/const.rb +5 -1
- data/lib/rainbows/dev_fd_response.rb +69 -0
- data/lib/rainbows/http_response.rb +1 -0
- data/lib/rainbows/http_server.rb +2 -0
- data/lib/rainbows/rev.rb +136 -43
- data/lib/rainbows/revactor.rb +6 -14
- data/lib/rainbows/thread_pool.rb +11 -13
- data/lib/rainbows/thread_spawn.rb +4 -4
- data/local.mk.sample +9 -1
- data/t/GNUmakefile +4 -4
- data/t/README +1 -1
- data/t/async-response-no-autochunk.ru +24 -0
- data/t/async-response.ru +13 -0
- data/t/bin/content-md5-put +1 -1
- data/t/bin/utee +12 -0
- data/t/env.ru +3 -0
- data/t/large-file-response.ru +13 -0
- data/t/lib-async-response-no-autochunk.sh +6 -0
- data/t/lib-async-response.sh +45 -0
- data/t/lib-graceful.sh +40 -0
- data/t/lib-input-trailer.sh +63 -0
- data/t/lib-large-file-response.sh +45 -0
- data/t/lib-parser-error.sh +29 -0
- data/t/{t3100-revactor-tee-input.sh → lib-rack-input-hammer.sh} +3 -9
- data/t/lib-reopen-logs.sh +60 -0
- data/t/lib-simple-http.sh +92 -0
- data/t/sleep.ru +13 -6
- data/t/t0000-basic.sh +1 -36
- data/t/t1000-thread-pool-basic.sh +1 -41
- data/t/t1002-thread-pool-graceful.sh +1 -36
- data/t/t1003-thread-pool-reopen-logs.sh +2 -0
- data/t/t1004-thread-pool-async-response.sh +45 -0
- data/t/t1005-thread-pool-large-file-response.sh +45 -0
- data/t/t1006-thread-pool-async-response-no-autochunk.sh +6 -0
- data/t/t1100-thread-pool-rack-input.sh +2 -0
- data/t/t1101-thread-pool-input-trailer.sh +2 -0
- data/t/t2000-thread-spawn-basic.sh +1 -37
- data/t/t2002-thread-spawn-graceful.sh +1 -36
- data/t/t2003-thread-spawn-reopen-logs.sh +2 -0
- data/t/t2004-thread-spawn-async-response.sh +45 -0
- data/t/t2005-thread-spawn-large-file-response.sh +45 -0
- data/t/t2006-thread-spawn-async-response-no-autochunk.sh +6 -0
- data/t/t2100-thread-spawn-rack-input.sh +2 -0
- data/t/t2101-thread-spawn-input-trailer.sh +2 -0
- data/t/t3000-revactor-basic.sh +1 -39
- data/t/t3002-revactor-graceful.sh +1 -37
- data/t/t3003-revactor-reopen-logs.sh +1 -53
- data/t/t3004-revactor-async-response.sh +45 -0
- data/t/t3005-revactor-large-file-response.sh +2 -0
- data/t/t3006-revactor-async-response-no-autochunk.sh +6 -0
- data/t/t3100-revactor-rack-input.sh +2 -0
- data/t/t3101-revactor-rack-input-trailer.sh +2 -0
- data/t/t4000-rev-basic.sh +1 -50
- data/t/t4002-rev-graceful.sh +1 -51
- data/t/t4003-rev-parser-error.sh +1 -33
- data/t/t4003-rev-reopen-logs.sh +2 -0
- data/t/t4004-rev-async-response.sh +45 -0
- data/t/t4005-rev-large-file-response.sh +2 -0
- data/t/t4006-rev-async-response-no-autochunk.sh +6 -0
- data/t/t4100-rev-rack-input.sh +1 -43
- data/t/t4101-rev-rack-input-trailer.sh +1 -50
- data/t/t9000-rack-app-pool.sh +2 -3
- data/t/test-lib.sh +80 -18
- metadata +39 -4
- data/t/t3001-revactor-pipeline.sh +0 -46
data/lib/rainbows/revactor.rb
CHANGED
@@ -55,7 +55,7 @@ module Rainbows
|
|
55
55
|
response = app.call(env)
|
56
56
|
end
|
57
57
|
|
58
|
-
alive = hp.keepalive? &&
|
58
|
+
alive = hp.keepalive? && G.alive
|
59
59
|
out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
|
60
60
|
HttpResponse.write(client, response, out)
|
61
61
|
end while alive and hp.reset.nil? and env.clear
|
@@ -86,7 +86,6 @@ module Rainbows
|
|
86
86
|
limit = worker_connections
|
87
87
|
revactorize_listeners!
|
88
88
|
clients = {}
|
89
|
-
alive = true
|
90
89
|
|
91
90
|
listeners = LISTENERS.map do |s|
|
92
91
|
Actor.spawn(s) do |l|
|
@@ -99,28 +98,21 @@ module Rainbows
|
|
99
98
|
clients[actor.object_id] = actor
|
100
99
|
root.link(actor)
|
101
100
|
rescue Errno::EAGAIN, Errno::ECONNABORTED
|
102
|
-
rescue Errno::EBADF
|
103
|
-
break
|
104
101
|
rescue Object => e
|
105
|
-
listen_loop_error(e)
|
106
|
-
end while alive
|
102
|
+
listen_loop_error(e)
|
103
|
+
end while G.alive
|
107
104
|
end
|
108
105
|
end
|
109
106
|
|
110
107
|
m = 0
|
111
108
|
check_quit = lambda do
|
112
109
|
worker.tmp.chmod(m = 0 == m ? 1 : 0)
|
113
|
-
|
114
|
-
master_pid != Process.ppid ||
|
115
|
-
LISTENERS.first.nil?
|
116
|
-
alive = false
|
117
|
-
clients.each_value { |a| a[:quit] = true }
|
118
|
-
end
|
110
|
+
G.alive = false if master_pid != Process.ppid
|
119
111
|
end
|
120
112
|
|
121
113
|
begin
|
122
114
|
Actor.receive do |filter|
|
123
|
-
filter.after(
|
115
|
+
filter.after(1, &check_quit)
|
124
116
|
filter.when(Case[:exit, Actor, Object]) do |_,actor,_|
|
125
117
|
orig = clients.size
|
126
118
|
clients.delete(actor.object_id)
|
@@ -128,7 +120,7 @@ module Rainbows
|
|
128
120
|
check_quit.call
|
129
121
|
end
|
130
122
|
end
|
131
|
-
end while alive || clients.size > 0
|
123
|
+
end while G.alive || clients.size > 0
|
132
124
|
end
|
133
125
|
|
134
126
|
private
|
data/lib/rainbows/thread_pool.rb
CHANGED
@@ -27,11 +27,10 @@ module Rainbows
|
|
27
27
|
|
28
28
|
def worker_loop(worker)
|
29
29
|
init_worker_process(worker)
|
30
|
-
RACK_DEFAULTS["rack.multithread"] = true
|
31
30
|
pool = (1..worker_connections).map { new_worker_thread }
|
32
31
|
m = 0
|
33
32
|
|
34
|
-
while
|
33
|
+
while G.alive && master_pid == Process.ppid
|
35
34
|
pool.each do |thr|
|
36
35
|
worker.tmp.chmod(m = 0 == m ? 1 : 0)
|
37
36
|
# if any worker dies, something is serious wrong, bail
|
@@ -45,21 +44,20 @@ module Rainbows
|
|
45
44
|
Thread.new {
|
46
45
|
begin
|
47
46
|
begin
|
48
|
-
ret = IO.select(LISTENERS, nil, nil,
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
47
|
+
ret = IO.select(LISTENERS, nil, nil, 1) and
|
48
|
+
ret.first.each do |sock|
|
49
|
+
begin
|
50
|
+
process_client(sock.accept_nonblock)
|
51
|
+
rescue Errno::EAGAIN, Errno::ECONNABORTED
|
52
|
+
end
|
53
|
+
end
|
55
54
|
rescue Errno::EINTR
|
56
|
-
next
|
57
55
|
rescue Errno::EBADF, TypeError
|
58
|
-
|
56
|
+
break
|
59
57
|
end
|
60
58
|
rescue Object => e
|
61
|
-
listen_loop_error(e)
|
62
|
-
end while
|
59
|
+
listen_loop_error(e)
|
60
|
+
end while G.alive
|
63
61
|
}
|
64
62
|
end
|
65
63
|
|
@@ -21,16 +21,16 @@ module Rainbows
|
|
21
21
|
|
22
22
|
def worker_loop(worker)
|
23
23
|
init_worker_process(worker)
|
24
|
-
RACK_DEFAULTS["rack.multithread"] = true
|
25
24
|
threads = ThreadGroup.new
|
26
25
|
alive = worker.tmp
|
27
26
|
m = 0
|
28
27
|
limit = worker_connections
|
29
28
|
|
30
29
|
begin
|
30
|
+
G.alive && master_pid == Process.ppid or break
|
31
31
|
ret = begin
|
32
32
|
alive.chmod(m = 0 == m ? 1 : 0)
|
33
|
-
IO.select(LISTENERS, nil, nil,
|
33
|
+
IO.select(LISTENERS, nil, nil, 1) or next
|
34
34
|
rescue Errno::EINTR
|
35
35
|
retry
|
36
36
|
rescue Errno::EBADF, TypeError
|
@@ -55,8 +55,8 @@ module Rainbows
|
|
55
55
|
end
|
56
56
|
end
|
57
57
|
rescue Object => e
|
58
|
-
listen_loop_error(e)
|
59
|
-
end while
|
58
|
+
listen_loop_error(e)
|
59
|
+
end while true
|
60
60
|
join_threads(threads.list, worker)
|
61
61
|
end
|
62
62
|
|
data/local.mk.sample
CHANGED
@@ -22,13 +22,17 @@ endif
|
|
22
22
|
ifdef gem_paths
|
23
23
|
sp :=
|
24
24
|
sp +=
|
25
|
-
RUBYLIB := $(subst $(sp),:,$(addsuffix /lib,$(gem_paths)))
|
25
|
+
export RUBYLIB := $(subst $(sp),:,$(addsuffix /lib,$(gem_paths)))
|
26
26
|
endif
|
27
27
|
|
28
28
|
# pipefail is THE reason to use bash (v3+) or never revisions of ksh93
|
29
29
|
# SHELL := /bin/bash -e -o pipefail
|
30
30
|
SHELL := /bin/ksh93 -e -o pipefail
|
31
31
|
|
32
|
+
# trace execution of tests
|
33
|
+
# TRACER = strace -f -o $(t_pfx).strace -s 100000
|
34
|
+
TRACER = /usr/bin/time -v -o $(t_pfx).time
|
35
|
+
|
32
36
|
full-test: test-18 test-19
|
33
37
|
test-18:
|
34
38
|
$(MAKE) test 2>&1 | sed -u -e 's!^!1.8 !'
|
@@ -60,3 +64,7 @@ doc_gz:
|
|
60
64
|
touch doc/NEWS.atom.xml -d "$$(awk 'NR==1{print $$4,$$5,$$6}' NEWS)"
|
61
65
|
for i in $(docs); do \
|
62
66
|
gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
|
67
|
+
|
68
|
+
# launches any of the following shells with RUBYLIB set
|
69
|
+
irb sh bash ksh:
|
70
|
+
$@
|
data/t/GNUmakefile
CHANGED
@@ -14,7 +14,7 @@ ifeq ($(RUBYLIB),)
|
|
14
14
|
else
|
15
15
|
RUBYLIB := $(rainbows_lib):$(RUBYLIB)
|
16
16
|
endif
|
17
|
-
export RUBYLIB
|
17
|
+
export RUBYLIB RUBY_VERSION
|
18
18
|
|
19
19
|
T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
|
20
20
|
|
@@ -44,7 +44,7 @@ else
|
|
44
44
|
SH_TEST_OPTS += -x
|
45
45
|
endif
|
46
46
|
quiet_pre = @echo '* $@';
|
47
|
-
quiet_post = 2>&1 |
|
47
|
+
quiet_post = 2>&1 | ./bin/utee $(t_log); exit $$(cat $(t_code))
|
48
48
|
pfx = $@
|
49
49
|
endif
|
50
50
|
|
@@ -58,12 +58,12 @@ test-bin-$(RUBY_VERSION)/rainbows: ../bin/rainbows
|
|
58
58
|
cmp $@+ $@ 2>/dev/null || mv $@+ $@
|
59
59
|
$(RM) $@+
|
60
60
|
|
61
|
-
req_random_blob :=
|
61
|
+
req_random_blob := $(wildcard t?1??-*.sh)
|
62
62
|
random_blob:
|
63
63
|
dd if=/dev/urandom bs=1M count=10 of=$@+
|
64
64
|
mv $@+ $@
|
65
65
|
|
66
|
-
$(
|
66
|
+
$(req_random_blob): random_blob
|
67
67
|
|
68
68
|
$(T): trash/.gitignore
|
69
69
|
$(T): export ruby := $(ruby)
|
data/t/README
CHANGED
@@ -14,7 +14,7 @@ comfortable writing integration tests with.
|
|
14
14
|
* {GNU make}[http://www.gnu.org/software/make/]
|
15
15
|
* {socat}[http://www.dest-unreach.org/socat/]
|
16
16
|
* {curl}[http://curl.haxx.se/]
|
17
|
-
* standard UNIX shell utilities (Bourne sh, awk, sed, grep,
|
17
|
+
* standard UNIX shell utilities (Bourne sh, awk, sed, grep, ...)
|
18
18
|
|
19
19
|
We do not use bashisms or any non-portable, non-POSIX constructs
|
20
20
|
in our shell code. We use the "pipefail" option if available and
|
@@ -0,0 +1,24 @@
|
|
1
|
+
use Rack::Chunked
|
2
|
+
use Rainbows::DevFdResponse
|
3
|
+
script = <<-EOF
|
4
|
+
for i in 0 1 2 3 4 5 6 7 8 9
|
5
|
+
do
|
6
|
+
printf '1\r\n%s\r\n' $i
|
7
|
+
sleep 1
|
8
|
+
done
|
9
|
+
printf '0\r\n\r\n'
|
10
|
+
EOF
|
11
|
+
|
12
|
+
run lambda { |env|
|
13
|
+
env['rainbows.autochunk'] = false
|
14
|
+
io = IO.popen(script, 'rb')
|
15
|
+
io.sync = true
|
16
|
+
[
|
17
|
+
200,
|
18
|
+
{
|
19
|
+
'Content-Type' => 'text/plain',
|
20
|
+
'Transfer-Encoding' => 'chunked',
|
21
|
+
},
|
22
|
+
io
|
23
|
+
].freeze
|
24
|
+
}
|
data/t/async-response.ru
ADDED
data/t/bin/content-md5-put
CHANGED
data/t/bin/utee
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# tee(1) as distributed on most(all?) systems is buffered in luserspace
|
3
|
+
# this only does unbuffered writes (with line-buffered input) to make
|
4
|
+
# test output appear in real-time
|
5
|
+
# -*- encoding: binary -*-
|
6
|
+
$stdin.binmode
|
7
|
+
$stdout.binmode
|
8
|
+
fp = File.open(ARGV.shift, "wb")
|
9
|
+
$stdin.each_line do |line|
|
10
|
+
fp.syswrite line
|
11
|
+
$stdout.syswrite line
|
12
|
+
end
|
data/t/env.ru
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# lib-large-file-response will stop running if we're not on Linux here
|
2
|
+
use Rack::ContentLength
|
3
|
+
use Rack::ContentType
|
4
|
+
map "/rss" do
|
5
|
+
run lambda { |env|
|
6
|
+
# on Linux, this is in kilobytes
|
7
|
+
::File.read("/proc/self/status") =~ /^VmRSS:\s+(\d+)/
|
8
|
+
[ 200, {}, [ ($1.to_i * 1024).to_s ] ]
|
9
|
+
}
|
10
|
+
end
|
11
|
+
map "/" do
|
12
|
+
run Rack::File.new(Dir.pwd)
|
13
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
CONFIG_RU=${CONFIG_RU-'async-response.ru'}
|
2
|
+
. ./test-lib.sh
|
3
|
+
echo "async response for model=$model"
|
4
|
+
eval $(unused_listen)
|
5
|
+
rtmpfiles unicorn_config a b c r_err r_out pid curl_err
|
6
|
+
|
7
|
+
cat > $unicorn_config <<EOF
|
8
|
+
listen "$listen"
|
9
|
+
stderr_path "$r_err"
|
10
|
+
stdout_path "$r_out"
|
11
|
+
pid "$pid"
|
12
|
+
Rainbows! { use :$model }
|
13
|
+
EOF
|
14
|
+
|
15
|
+
# can't load Rack::Lint here since it'll cause Rev to slurp
|
16
|
+
rainbows -E none -D $CONFIG_RU -c $unicorn_config
|
17
|
+
wait_for_pid $pid
|
18
|
+
|
19
|
+
t0=$(date +%s)
|
20
|
+
( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $a) &
|
21
|
+
( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $b) &
|
22
|
+
( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $c) &
|
23
|
+
wait
|
24
|
+
t1=$(date +%s)
|
25
|
+
|
26
|
+
rainbows_pid=$(cat $pid)
|
27
|
+
kill -QUIT $rainbows_pid
|
28
|
+
elapsed=$(( $t1 - $t0 ))
|
29
|
+
echo "elapsed=$elapsed < 30"
|
30
|
+
test $elapsed -lt 30
|
31
|
+
|
32
|
+
dbgcat a
|
33
|
+
dbgcat b
|
34
|
+
dbgcat c
|
35
|
+
dbgcat r_err
|
36
|
+
dbgcat curl_err
|
37
|
+
test ! -s $curl_err
|
38
|
+
check_stderr
|
39
|
+
|
40
|
+
while kill -0 $rainbows_pid >/dev/null 2>&1
|
41
|
+
do
|
42
|
+
sleep 1
|
43
|
+
done
|
44
|
+
|
45
|
+
dbgcat r_err
|
data/t/lib-graceful.sh
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
. ./test-lib.sh
|
2
|
+
echo "graceful test for model=$model"
|
3
|
+
|
4
|
+
eval $(unused_listen)
|
5
|
+
rtmpfiles unicorn_config curl_out pid r_err r_out fifo
|
6
|
+
|
7
|
+
cat > $unicorn_config <<EOF
|
8
|
+
listen "$listen"
|
9
|
+
stderr_path "$r_err"
|
10
|
+
stdout_path "$r_out"
|
11
|
+
pid "$pid"
|
12
|
+
Rainbows! { use :$model }
|
13
|
+
EOF
|
14
|
+
|
15
|
+
rainbows -D sleep.ru -c $unicorn_config
|
16
|
+
wait_for_pid $pid
|
17
|
+
rainbows_pid=$(cat $pid)
|
18
|
+
|
19
|
+
curl -sSfv -T- </dev/null http://$listen/5 > $curl_out 2> $fifo &
|
20
|
+
|
21
|
+
awk -v rainbows_pid=$rainbows_pid '
|
22
|
+
{ print $0 }
|
23
|
+
/100 Continue/ {
|
24
|
+
print "awk: sending SIGQUIT to", rainbows_pid
|
25
|
+
system("kill -QUIT "rainbows_pid)
|
26
|
+
}' $fifo
|
27
|
+
wait
|
28
|
+
|
29
|
+
dbgcat r_err
|
30
|
+
|
31
|
+
test x"$(wc -l < $curl_out)" = x1
|
32
|
+
nr=$(sort < $curl_out | uniq | wc -l)
|
33
|
+
|
34
|
+
test "$nr" -eq 1
|
35
|
+
test x$(sort < $curl_out | uniq) = xHello
|
36
|
+
check_stderr
|
37
|
+
while kill -0 $rainbows_pid >/dev/null 2>&1
|
38
|
+
do
|
39
|
+
sleep 1
|
40
|
+
done
|
@@ -0,0 +1,63 @@
|
|
1
|
+
. ./test-lib.sh
|
2
|
+
test -r random_blob || die "random_blob required, run with 'make $0'"
|
3
|
+
echo "input trailer test model=$model"
|
4
|
+
|
5
|
+
eval $(unused_listen)
|
6
|
+
rtmpfiles unicorn_config tmp r_err r_out pid fifo ok
|
7
|
+
|
8
|
+
cat > $unicorn_config <<EOF
|
9
|
+
listen "$listen"
|
10
|
+
pid "$pid"
|
11
|
+
stderr_path "$r_err"
|
12
|
+
stdout_path "$r_out"
|
13
|
+
Rainbows! { use :$model }
|
14
|
+
EOF
|
15
|
+
|
16
|
+
rainbows -D content-md5.ru -c $unicorn_config
|
17
|
+
wait_for_pid $pid
|
18
|
+
|
19
|
+
echo "small blob"
|
20
|
+
(
|
21
|
+
echo hello world | content-md5-put
|
22
|
+
cat $fifo > $tmp &
|
23
|
+
wait
|
24
|
+
echo ok > $ok
|
25
|
+
) | socat - TCP:$listen | utee $fifo
|
26
|
+
|
27
|
+
fgrep 'HTTP/1.1 200 OK' $tmp
|
28
|
+
test xok = x"$(cat $ok)"
|
29
|
+
check_stderr
|
30
|
+
|
31
|
+
echo "big blob"
|
32
|
+
(
|
33
|
+
content-md5-put < random_blob
|
34
|
+
cat $fifo > $tmp &
|
35
|
+
wait
|
36
|
+
echo ok > $ok
|
37
|
+
) | socat - TCP:$listen | utee $fifo
|
38
|
+
|
39
|
+
fgrep 'HTTP/1.1 200 OK' $tmp
|
40
|
+
test xok = x"$(cat $ok)"
|
41
|
+
check_stderr
|
42
|
+
|
43
|
+
echo "staggered blob"
|
44
|
+
(
|
45
|
+
(
|
46
|
+
dd bs=164 count=1 < random_blob
|
47
|
+
sleep 2
|
48
|
+
dd bs=4545 count=1 < random_blob
|
49
|
+
sleep 2
|
50
|
+
dd bs=1234 count=1 < random_blob
|
51
|
+
echo ok > $ok
|
52
|
+
) 2>/dev/null | content-md5-put
|
53
|
+
test xok = x"$(cat $ok)"
|
54
|
+
cat $fifo > $tmp &
|
55
|
+
wait
|
56
|
+
echo ok > $ok
|
57
|
+
) | socat - TCP:$listen | utee $fifo
|
58
|
+
|
59
|
+
fgrep 'HTTP/1.1 200 OK' $tmp
|
60
|
+
test xok = x"$(cat $ok)"
|
61
|
+
check_stderr
|
62
|
+
|
63
|
+
kill $(cat $pid)
|