rainbows 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. data/GIT-VERSION-GEN +1 -1
  2. data/TODO +15 -0
  3. data/TUNING +14 -0
  4. data/lib/rainbows.rb +7 -0
  5. data/lib/rainbows/app_pool.rb +4 -3
  6. data/lib/rainbows/base.rb +17 -11
  7. data/lib/rainbows/const.rb +5 -1
  8. data/lib/rainbows/dev_fd_response.rb +69 -0
  9. data/lib/rainbows/http_response.rb +1 -0
  10. data/lib/rainbows/http_server.rb +2 -0
  11. data/lib/rainbows/rev.rb +136 -43
  12. data/lib/rainbows/revactor.rb +6 -14
  13. data/lib/rainbows/thread_pool.rb +11 -13
  14. data/lib/rainbows/thread_spawn.rb +4 -4
  15. data/local.mk.sample +9 -1
  16. data/t/GNUmakefile +4 -4
  17. data/t/README +1 -1
  18. data/t/async-response-no-autochunk.ru +24 -0
  19. data/t/async-response.ru +13 -0
  20. data/t/bin/content-md5-put +1 -1
  21. data/t/bin/utee +12 -0
  22. data/t/env.ru +3 -0
  23. data/t/large-file-response.ru +13 -0
  24. data/t/lib-async-response-no-autochunk.sh +6 -0
  25. data/t/lib-async-response.sh +45 -0
  26. data/t/lib-graceful.sh +40 -0
  27. data/t/lib-input-trailer.sh +63 -0
  28. data/t/lib-large-file-response.sh +45 -0
  29. data/t/lib-parser-error.sh +29 -0
  30. data/t/{t3100-revactor-tee-input.sh → lib-rack-input-hammer.sh} +3 -9
  31. data/t/lib-reopen-logs.sh +60 -0
  32. data/t/lib-simple-http.sh +92 -0
  33. data/t/sleep.ru +13 -6
  34. data/t/t0000-basic.sh +1 -36
  35. data/t/t1000-thread-pool-basic.sh +1 -41
  36. data/t/t1002-thread-pool-graceful.sh +1 -36
  37. data/t/t1003-thread-pool-reopen-logs.sh +2 -0
  38. data/t/t1004-thread-pool-async-response.sh +45 -0
  39. data/t/t1005-thread-pool-large-file-response.sh +45 -0
  40. data/t/t1006-thread-pool-async-response-no-autochunk.sh +6 -0
  41. data/t/t1100-thread-pool-rack-input.sh +2 -0
  42. data/t/t1101-thread-pool-input-trailer.sh +2 -0
  43. data/t/t2000-thread-spawn-basic.sh +1 -37
  44. data/t/t2002-thread-spawn-graceful.sh +1 -36
  45. data/t/t2003-thread-spawn-reopen-logs.sh +2 -0
  46. data/t/t2004-thread-spawn-async-response.sh +45 -0
  47. data/t/t2005-thread-spawn-large-file-response.sh +45 -0
  48. data/t/t2006-thread-spawn-async-response-no-autochunk.sh +6 -0
  49. data/t/t2100-thread-spawn-rack-input.sh +2 -0
  50. data/t/t2101-thread-spawn-input-trailer.sh +2 -0
  51. data/t/t3000-revactor-basic.sh +1 -39
  52. data/t/t3002-revactor-graceful.sh +1 -37
  53. data/t/t3003-revactor-reopen-logs.sh +1 -53
  54. data/t/t3004-revactor-async-response.sh +45 -0
  55. data/t/t3005-revactor-large-file-response.sh +2 -0
  56. data/t/t3006-revactor-async-response-no-autochunk.sh +6 -0
  57. data/t/t3100-revactor-rack-input.sh +2 -0
  58. data/t/t3101-revactor-rack-input-trailer.sh +2 -0
  59. data/t/t4000-rev-basic.sh +1 -50
  60. data/t/t4002-rev-graceful.sh +1 -51
  61. data/t/t4003-rev-parser-error.sh +1 -33
  62. data/t/t4003-rev-reopen-logs.sh +2 -0
  63. data/t/t4004-rev-async-response.sh +45 -0
  64. data/t/t4005-rev-large-file-response.sh +2 -0
  65. data/t/t4006-rev-async-response-no-autochunk.sh +6 -0
  66. data/t/t4100-rev-rack-input.sh +1 -43
  67. data/t/t4101-rev-rack-input-trailer.sh +1 -50
  68. data/t/t9000-rack-app-pool.sh +2 -3
  69. data/t/test-lib.sh +80 -18
  70. metadata +39 -4
  71. data/t/t3001-revactor-pipeline.sh +0 -46
@@ -55,7 +55,7 @@ module Rainbows
55
55
  response = app.call(env)
56
56
  end
57
57
 
58
- alive = hp.keepalive? && ! Actor.current[:quit]
58
+ alive = hp.keepalive? && G.alive
59
59
  out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
60
60
  HttpResponse.write(client, response, out)
61
61
  end while alive and hp.reset.nil? and env.clear
@@ -86,7 +86,6 @@ module Rainbows
86
86
  limit = worker_connections
87
87
  revactorize_listeners!
88
88
  clients = {}
89
- alive = true
90
89
 
91
90
  listeners = LISTENERS.map do |s|
92
91
  Actor.spawn(s) do |l|
@@ -99,28 +98,21 @@ module Rainbows
99
98
  clients[actor.object_id] = actor
100
99
  root.link(actor)
101
100
  rescue Errno::EAGAIN, Errno::ECONNABORTED
102
- rescue Errno::EBADF
103
- break
104
101
  rescue Object => e
105
- listen_loop_error(e) if alive
106
- end while alive
102
+ listen_loop_error(e)
103
+ end while G.alive
107
104
  end
108
105
  end
109
106
 
110
107
  m = 0
111
108
  check_quit = lambda do
112
109
  worker.tmp.chmod(m = 0 == m ? 1 : 0)
113
- if listeners.any? { |l| l.dead? } ||
114
- master_pid != Process.ppid ||
115
- LISTENERS.first.nil?
116
- alive = false
117
- clients.each_value { |a| a[:quit] = true }
118
- end
110
+ G.alive = false if master_pid != Process.ppid
119
111
  end
120
112
 
121
113
  begin
122
114
  Actor.receive do |filter|
123
- filter.after(timeout, &check_quit)
115
+ filter.after(1, &check_quit)
124
116
  filter.when(Case[:exit, Actor, Object]) do |_,actor,_|
125
117
  orig = clients.size
126
118
  clients.delete(actor.object_id)
@@ -128,7 +120,7 @@ module Rainbows
128
120
  check_quit.call
129
121
  end
130
122
  end
131
- end while alive || clients.size > 0
123
+ end while G.alive || clients.size > 0
132
124
  end
133
125
 
134
126
  private
@@ -27,11 +27,10 @@ module Rainbows
27
27
 
28
28
  def worker_loop(worker)
29
29
  init_worker_process(worker)
30
- RACK_DEFAULTS["rack.multithread"] = true
31
30
  pool = (1..worker_connections).map { new_worker_thread }
32
31
  m = 0
33
32
 
34
- while LISTENERS.first && master_pid == Process.ppid
33
+ while G.alive && master_pid == Process.ppid
35
34
  pool.each do |thr|
36
35
  worker.tmp.chmod(m = 0 == m ? 1 : 0)
37
36
  # if any worker dies, something is serious wrong, bail
@@ -45,21 +44,20 @@ module Rainbows
45
44
  Thread.new {
46
45
  begin
47
46
  begin
48
- ret = IO.select(LISTENERS, nil, nil, timeout) or next
49
- ret.first.each do |sock|
50
- begin
51
- process_client(sock.accept_nonblock)
52
- rescue Errno::EAGAIN, Errno::ECONNABORTED
53
- end
54
- end
47
+ ret = IO.select(LISTENERS, nil, nil, 1) and
48
+ ret.first.each do |sock|
49
+ begin
50
+ process_client(sock.accept_nonblock)
51
+ rescue Errno::EAGAIN, Errno::ECONNABORTED
52
+ end
53
+ end
55
54
  rescue Errno::EINTR
56
- next
57
55
  rescue Errno::EBADF, TypeError
58
- return
56
+ break
59
57
  end
60
58
  rescue Object => e
61
- listen_loop_error(e) if LISTENERS.first
62
- end while ! Thread.current[:quit] && LISTENERS.first
59
+ listen_loop_error(e)
60
+ end while G.alive
63
61
  }
64
62
  end
65
63
 
@@ -21,16 +21,16 @@ module Rainbows
21
21
 
22
22
  def worker_loop(worker)
23
23
  init_worker_process(worker)
24
- RACK_DEFAULTS["rack.multithread"] = true
25
24
  threads = ThreadGroup.new
26
25
  alive = worker.tmp
27
26
  m = 0
28
27
  limit = worker_connections
29
28
 
30
29
  begin
30
+ G.alive && master_pid == Process.ppid or break
31
31
  ret = begin
32
32
  alive.chmod(m = 0 == m ? 1 : 0)
33
- IO.select(LISTENERS, nil, nil, timeout) or next
33
+ IO.select(LISTENERS, nil, nil, 1) or next
34
34
  rescue Errno::EINTR
35
35
  retry
36
36
  rescue Errno::EBADF, TypeError
@@ -55,8 +55,8 @@ module Rainbows
55
55
  end
56
56
  end
57
57
  rescue Object => e
58
- listen_loop_error(e) if LISTENERS.first
59
- end while LISTENERS.first && master_pid == Process.ppid
58
+ listen_loop_error(e)
59
+ end while true
60
60
  join_threads(threads.list, worker)
61
61
  end
62
62
 
data/local.mk.sample CHANGED
@@ -22,13 +22,17 @@ endif
22
22
  ifdef gem_paths
23
23
  sp :=
24
24
  sp +=
25
- RUBYLIB := $(subst $(sp),:,$(addsuffix /lib,$(gem_paths)))
25
+ export RUBYLIB := $(subst $(sp),:,$(addsuffix /lib,$(gem_paths)))
26
26
  endif
27
27
 
28
28
  # pipefail is THE reason to use bash (v3+) or never revisions of ksh93
29
29
  # SHELL := /bin/bash -e -o pipefail
30
30
  SHELL := /bin/ksh93 -e -o pipefail
31
31
 
32
+ # trace execution of tests
33
+ # TRACER = strace -f -o $(t_pfx).strace -s 100000
34
+ TRACER = /usr/bin/time -v -o $(t_pfx).time
35
+
32
36
  full-test: test-18 test-19
33
37
  test-18:
34
38
  $(MAKE) test 2>&1 | sed -u -e 's!^!1.8 !'
@@ -60,3 +64,7 @@ doc_gz:
60
64
  touch doc/NEWS.atom.xml -d "$$(awk 'NR==1{print $$4,$$5,$$6}' NEWS)"
61
65
  for i in $(docs); do \
62
66
  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
67
+
68
+ # launches any of the following shells with RUBYLIB set
69
+ irb sh bash ksh:
70
+ $@
data/t/GNUmakefile CHANGED
@@ -14,7 +14,7 @@ ifeq ($(RUBYLIB),)
14
14
  else
15
15
  RUBYLIB := $(rainbows_lib):$(RUBYLIB)
16
16
  endif
17
- export RUBYLIB
17
+ export RUBYLIB RUBY_VERSION
18
18
 
19
19
  T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
20
20
 
@@ -44,7 +44,7 @@ else
44
44
  SH_TEST_OPTS += -x
45
45
  endif
46
46
  quiet_pre = @echo '* $@';
47
- quiet_post = 2>&1 | tee $(t_log); exit $$(cat $(t_code))
47
+ quiet_post = 2>&1 | ./bin/utee $(t_log); exit $$(cat $(t_code))
48
48
  pfx = $@
49
49
  endif
50
50
 
@@ -58,12 +58,12 @@ test-bin-$(RUBY_VERSION)/rainbows: ../bin/rainbows
58
58
  cmp $@+ $@ 2>/dev/null || mv $@+ $@
59
59
  $(RM) $@+
60
60
 
61
- req_random_blob := t3100-revactor-tee-input
61
+ req_random_blob := $(wildcard t?1??-*.sh)
62
62
  random_blob:
63
63
  dd if=/dev/urandom bs=1M count=10 of=$@+
64
64
  mv $@+ $@
65
65
 
66
- $(addsuffix .sh,$(req_random_blob)): random_blob
66
+ $(req_random_blob): random_blob
67
67
 
68
68
  $(T): trash/.gitignore
69
69
  $(T): export ruby := $(ruby)
data/t/README CHANGED
@@ -14,7 +14,7 @@ comfortable writing integration tests with.
14
14
  * {GNU make}[http://www.gnu.org/software/make/]
15
15
  * {socat}[http://www.dest-unreach.org/socat/]
16
16
  * {curl}[http://curl.haxx.se/]
17
- * standard UNIX shell utilities (Bourne sh, awk, sed, grep, tee, ...)
17
+ * standard UNIX shell utilities (Bourne sh, awk, sed, grep, ...)
18
18
 
19
19
  We do not use bashisms or any non-portable, non-POSIX constructs
20
20
  in our shell code. We use the "pipefail" option if available and
@@ -0,0 +1,24 @@
1
+ use Rack::Chunked
2
+ use Rainbows::DevFdResponse
3
+ script = <<-EOF
4
+ for i in 0 1 2 3 4 5 6 7 8 9
5
+ do
6
+ printf '1\r\n%s\r\n' $i
7
+ sleep 1
8
+ done
9
+ printf '0\r\n\r\n'
10
+ EOF
11
+
12
+ run lambda { |env|
13
+ env['rainbows.autochunk'] = false
14
+ io = IO.popen(script, 'rb')
15
+ io.sync = true
16
+ [
17
+ 200,
18
+ {
19
+ 'Content-Type' => 'text/plain',
20
+ 'Transfer-Encoding' => 'chunked',
21
+ },
22
+ io
23
+ ].freeze
24
+ }
@@ -0,0 +1,13 @@
1
+ use Rack::Chunked
2
+ use Rainbows::DevFdResponse
3
+ run lambda { |env|
4
+ io = IO.popen('for i in 0 1 2 3 4 5 6 7 8 9; do date; sleep 1; done', 'rb')
5
+ io.sync = true
6
+ [
7
+ 200,
8
+ {
9
+ 'Content-Type' => 'text/plain',
10
+ },
11
+ io
12
+ ].freeze
13
+ }
@@ -22,7 +22,7 @@ if ARGV.grep("--no-headers").empty?
22
22
  end
23
23
 
24
24
  digest = Digest::MD5.new
25
- if buf = $stdin.read(bs)
25
+ if buf = $stdin.readpartial(bs)
26
26
  begin
27
27
  digest.update(buf)
28
28
  $stdout.write("%x\r\n" % [ buf.size ])
data/t/bin/utee ADDED
@@ -0,0 +1,12 @@
1
+ #!/usr/bin/env ruby
2
+ # tee(1) as distributed on most(all?) systems is buffered in luserspace
3
+ # this only does unbuffered writes (with line-buffered input) to make
4
+ # test output appear in real-time
5
+ # -*- encoding: binary -*-
6
+ $stdin.binmode
7
+ $stdout.binmode
8
+ fp = File.open(ARGV.shift, "wb")
9
+ $stdin.each_line do |line|
10
+ fp.syswrite line
11
+ $stdout.syswrite line
12
+ end
data/t/env.ru ADDED
@@ -0,0 +1,3 @@
1
+ use Rack::ContentLength
2
+ use Rack::ContentType
3
+ run lambda { |env| [ 200, {}, [ env.inspect << "\n" ] ] }
@@ -0,0 +1,13 @@
1
+ # lib-large-file-response will stop running if we're not on Linux here
2
+ use Rack::ContentLength
3
+ use Rack::ContentType
4
+ map "/rss" do
5
+ run lambda { |env|
6
+ # on Linux, this is in kilobytes
7
+ ::File.read("/proc/self/status") =~ /^VmRSS:\s+(\d+)/
8
+ [ 200, {}, [ ($1.to_i * 1024).to_s ] ]
9
+ }
10
+ end
11
+ map "/" do
12
+ run Rack::File.new(Dir.pwd)
13
+ end
@@ -0,0 +1,6 @@
1
+ #!/bin/sh
2
+ CONFIG_RU=async-response-no-autochunk.ru
3
+ . ./lib-async-response.sh
4
+ test x"$(cat $a)" = x0123456789
5
+ test x"$(cat $b)" = x0123456789
6
+ test x"$(cat $c)" = x0123456789
@@ -0,0 +1,45 @@
1
+ CONFIG_RU=${CONFIG_RU-'async-response.ru'}
2
+ . ./test-lib.sh
3
+ echo "async response for model=$model"
4
+ eval $(unused_listen)
5
+ rtmpfiles unicorn_config a b c r_err r_out pid curl_err
6
+
7
+ cat > $unicorn_config <<EOF
8
+ listen "$listen"
9
+ stderr_path "$r_err"
10
+ stdout_path "$r_out"
11
+ pid "$pid"
12
+ Rainbows! { use :$model }
13
+ EOF
14
+
15
+ # can't load Rack::Lint here since it'll cause Rev to slurp
16
+ rainbows -E none -D $CONFIG_RU -c $unicorn_config
17
+ wait_for_pid $pid
18
+
19
+ t0=$(date +%s)
20
+ ( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $a) &
21
+ ( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $b) &
22
+ ( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $c) &
23
+ wait
24
+ t1=$(date +%s)
25
+
26
+ rainbows_pid=$(cat $pid)
27
+ kill -QUIT $rainbows_pid
28
+ elapsed=$(( $t1 - $t0 ))
29
+ echo "elapsed=$elapsed < 30"
30
+ test $elapsed -lt 30
31
+
32
+ dbgcat a
33
+ dbgcat b
34
+ dbgcat c
35
+ dbgcat r_err
36
+ dbgcat curl_err
37
+ test ! -s $curl_err
38
+ check_stderr
39
+
40
+ while kill -0 $rainbows_pid >/dev/null 2>&1
41
+ do
42
+ sleep 1
43
+ done
44
+
45
+ dbgcat r_err
data/t/lib-graceful.sh ADDED
@@ -0,0 +1,40 @@
1
+ . ./test-lib.sh
2
+ echo "graceful test for model=$model"
3
+
4
+ eval $(unused_listen)
5
+ rtmpfiles unicorn_config curl_out pid r_err r_out fifo
6
+
7
+ cat > $unicorn_config <<EOF
8
+ listen "$listen"
9
+ stderr_path "$r_err"
10
+ stdout_path "$r_out"
11
+ pid "$pid"
12
+ Rainbows! { use :$model }
13
+ EOF
14
+
15
+ rainbows -D sleep.ru -c $unicorn_config
16
+ wait_for_pid $pid
17
+ rainbows_pid=$(cat $pid)
18
+
19
+ curl -sSfv -T- </dev/null http://$listen/5 > $curl_out 2> $fifo &
20
+
21
+ awk -v rainbows_pid=$rainbows_pid '
22
+ { print $0 }
23
+ /100 Continue/ {
24
+ print "awk: sending SIGQUIT to", rainbows_pid
25
+ system("kill -QUIT "rainbows_pid)
26
+ }' $fifo
27
+ wait
28
+
29
+ dbgcat r_err
30
+
31
+ test x"$(wc -l < $curl_out)" = x1
32
+ nr=$(sort < $curl_out | uniq | wc -l)
33
+
34
+ test "$nr" -eq 1
35
+ test x$(sort < $curl_out | uniq) = xHello
36
+ check_stderr
37
+ while kill -0 $rainbows_pid >/dev/null 2>&1
38
+ do
39
+ sleep 1
40
+ done
@@ -0,0 +1,63 @@
1
+ . ./test-lib.sh
2
+ test -r random_blob || die "random_blob required, run with 'make $0'"
3
+ echo "input trailer test model=$model"
4
+
5
+ eval $(unused_listen)
6
+ rtmpfiles unicorn_config tmp r_err r_out pid fifo ok
7
+
8
+ cat > $unicorn_config <<EOF
9
+ listen "$listen"
10
+ pid "$pid"
11
+ stderr_path "$r_err"
12
+ stdout_path "$r_out"
13
+ Rainbows! { use :$model }
14
+ EOF
15
+
16
+ rainbows -D content-md5.ru -c $unicorn_config
17
+ wait_for_pid $pid
18
+
19
+ echo "small blob"
20
+ (
21
+ echo hello world | content-md5-put
22
+ cat $fifo > $tmp &
23
+ wait
24
+ echo ok > $ok
25
+ ) | socat - TCP:$listen | utee $fifo
26
+
27
+ fgrep 'HTTP/1.1 200 OK' $tmp
28
+ test xok = x"$(cat $ok)"
29
+ check_stderr
30
+
31
+ echo "big blob"
32
+ (
33
+ content-md5-put < random_blob
34
+ cat $fifo > $tmp &
35
+ wait
36
+ echo ok > $ok
37
+ ) | socat - TCP:$listen | utee $fifo
38
+
39
+ fgrep 'HTTP/1.1 200 OK' $tmp
40
+ test xok = x"$(cat $ok)"
41
+ check_stderr
42
+
43
+ echo "staggered blob"
44
+ (
45
+ (
46
+ dd bs=164 count=1 < random_blob
47
+ sleep 2
48
+ dd bs=4545 count=1 < random_blob
49
+ sleep 2
50
+ dd bs=1234 count=1 < random_blob
51
+ echo ok > $ok
52
+ ) 2>/dev/null | content-md5-put
53
+ test xok = x"$(cat $ok)"
54
+ cat $fifo > $tmp &
55
+ wait
56
+ echo ok > $ok
57
+ ) | socat - TCP:$listen | utee $fifo
58
+
59
+ fgrep 'HTTP/1.1 200 OK' $tmp
60
+ test xok = x"$(cat $ok)"
61
+ check_stderr
62
+
63
+ kill $(cat $pid)