rainbows 0.2.0 → 0.3.0

data/lib/rainbows/revactor.rb

@@ -55,7 +55,7 @@ module Rainbows
           response = app.call(env)
         end
 
-        alive = hp.keepalive? && ! Actor.current[:quit]
+        alive = hp.keepalive? && G.alive
         out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
         HttpResponse.write(client, response, out)
       end while alive and hp.reset.nil? and env.clear
@@ -86,7 +86,6 @@ module Rainbows
       limit = worker_connections
       revactorize_listeners!
       clients = {}
-      alive = true
 
       listeners = LISTENERS.map do |s|
         Actor.spawn(s) do |l|
@@ -99,28 +98,21 @@ module Rainbows
             clients[actor.object_id] = actor
             root.link(actor)
           rescue Errno::EAGAIN, Errno::ECONNABORTED
-          rescue Errno::EBADF
-            break
           rescue Object => e
-            listen_loop_error(e) if alive
-          end while alive
+            listen_loop_error(e)
+          end while G.alive
         end
       end
 
       m = 0
       check_quit = lambda do
         worker.tmp.chmod(m = 0 == m ? 1 : 0)
-        if listeners.any? { |l| l.dead? } ||
-           master_pid != Process.ppid ||
-           LISTENERS.first.nil?
-          alive = false
-          clients.each_value { |a| a[:quit] = true }
-        end
+        G.alive = false if master_pid != Process.ppid
       end
 
       begin
         Actor.receive do |filter|
-          filter.after(timeout, &check_quit)
+          filter.after(1, &check_quit)
           filter.when(Case[:exit, Actor, Object]) do |_,actor,_|
             orig = clients.size
             clients.delete(actor.object_id)
@@ -128,7 +120,7 @@ module Rainbows
             check_quit.call
           end
         end
-      end while alive || clients.size > 0
+      end while G.alive || clients.size > 0
     end
 
   private

data/lib/rainbows/thread_pool.rb

@@ -27,11 +27,10 @@ module Rainbows
 
     def worker_loop(worker)
       init_worker_process(worker)
-      RACK_DEFAULTS["rack.multithread"] = true
       pool = (1..worker_connections).map { new_worker_thread }
       m = 0
 
-      while LISTENERS.first && master_pid == Process.ppid
+      while G.alive && master_pid == Process.ppid
         pool.each do |thr|
           worker.tmp.chmod(m = 0 == m ? 1 : 0)
           # if any worker dies, something is serious wrong, bail
@@ -45,21 +44,20 @@ module Rainbows
       Thread.new {
         begin
           begin
-            ret = IO.select(LISTENERS, nil, nil, timeout) or next
-            ret.first.each do |sock|
-              begin
-                process_client(sock.accept_nonblock)
-              rescue Errno::EAGAIN, Errno::ECONNABORTED
-              end
-            end
+            ret = IO.select(LISTENERS, nil, nil, 1) and
+                  ret.first.each do |sock|
+                    begin
+                      process_client(sock.accept_nonblock)
+                    rescue Errno::EAGAIN, Errno::ECONNABORTED
+                    end
+                  end
           rescue Errno::EINTR
-            next
           rescue Errno::EBADF, TypeError
-            return
+            break
           end
         rescue Object => e
-          listen_loop_error(e) if LISTENERS.first
-        end while ! Thread.current[:quit] && LISTENERS.first
+          listen_loop_error(e)
+        end while G.alive
       }
     end
 

data/lib/rainbows/thread_spawn.rb

@@ -21,16 +21,16 @@ module Rainbows
 
     def worker_loop(worker)
       init_worker_process(worker)
-      RACK_DEFAULTS["rack.multithread"] = true
       threads = ThreadGroup.new
       alive = worker.tmp
       m = 0
       limit = worker_connections
 
       begin
+        G.alive && master_pid == Process.ppid or break
         ret = begin
           alive.chmod(m = 0 == m ? 1 : 0)
-          IO.select(LISTENERS, nil, nil, timeout) or next
+          IO.select(LISTENERS, nil, nil, 1) or next
         rescue Errno::EINTR
           retry
         rescue Errno::EBADF, TypeError
@@ -55,8 +55,8 @@ module Rainbows
           end
         end
       rescue Object => e
-        listen_loop_error(e) if LISTENERS.first
-      end while LISTENERS.first && master_pid == Process.ppid
+        listen_loop_error(e)
+      end while true
       join_threads(threads.list, worker)
     end
 

data/local.mk.sample

@@ -22,13 +22,17 @@ endif
 ifdef gem_paths
   sp :=
   sp +=
-  RUBYLIB := $(subst $(sp),:,$(addsuffix /lib,$(gem_paths)))
+  export RUBYLIB := $(subst $(sp),:,$(addsuffix /lib,$(gem_paths)))
 endif
 
 # pipefail is THE reason to use bash (v3+) or never revisions of ksh93
 # SHELL := /bin/bash -e -o pipefail
 SHELL := /bin/ksh93 -e -o pipefail
 
+# trace execution of tests
+# TRACER = strace -f -o $(t_pfx).strace -s 100000
+TRACER = /usr/bin/time -v -o $(t_pfx).time
+
 full-test: test-18 test-19
 test-18:
 	$(MAKE) test 2>&1 | sed -u -e 's!^!1.8 !'
@@ -60,3 +64,7 @@ doc_gz:
 	touch doc/NEWS.atom.xml -d "$$(awk 'NR==1{print $$4,$$5,$$6}' NEWS)"
 	for i in $(docs); do \
 	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
+
+# launches any of the following shells with RUBYLIB set
+irb sh bash ksh:
+	$@

data/t/GNUmakefile

@@ -14,7 +14,7 @@ ifeq ($(RUBYLIB),)
 else
   RUBYLIB := $(rainbows_lib):$(RUBYLIB)
 endif
-export RUBYLIB
+export RUBYLIB RUBY_VERSION
 
 T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
 
@@ -44,7 +44,7 @@ else
     SH_TEST_OPTS += -x
   endif
   quiet_pre = @echo '* $@';
-  quiet_post = 2>&1 | tee $(t_log); exit $$(cat $(t_code))
+  quiet_post = 2>&1 | ./bin/utee $(t_log); exit $$(cat $(t_code))
   pfx = $@
 endif
 
@@ -58,12 +58,12 @@ test-bin-$(RUBY_VERSION)/rainbows: ../bin/rainbows
 	cmp $@+ $@ 2>/dev/null || mv $@+ $@
 	$(RM) $@+
 
-req_random_blob := t3100-revactor-tee-input
+req_random_blob := $(wildcard t?1??-*.sh)
 random_blob:
 	dd if=/dev/urandom bs=1M count=10 of=$@+
 	mv $@+ $@
 
-$(addsuffix .sh,$(req_random_blob)): random_blob
+$(req_random_blob): random_blob
 
 $(T): trash/.gitignore
 $(T): export ruby := $(ruby)

data/t/README

@@ -14,7 +14,7 @@ comfortable writing integration tests with.
 * {GNU make}[http://www.gnu.org/software/make/]
 * {socat}[http://www.dest-unreach.org/socat/]
 * {curl}[http://curl.haxx.se/]
-* standard UNIX shell utilities (Bourne sh, awk, sed, grep, tee, ...)
+* standard UNIX shell utilities (Bourne sh, awk, sed, grep, ...)
 
 We do not use bashisms or any non-portable, non-POSIX constructs
 in our shell code.  We use the "pipefail" option if available and

data/t/async-response-no-autochunk.ru

@@ -0,0 +1,24 @@
+use Rack::Chunked
+use Rainbows::DevFdResponse
+script = <<-EOF
+for i in 0 1 2 3 4 5 6 7 8 9
+do
+	printf '1\r\n%s\r\n' $i
+	sleep 1
+done
+printf '0\r\n\r\n'
+EOF
+
+run lambda { |env|
+  env['rainbows.autochunk'] = false
+  io = IO.popen(script, 'rb')
+  io.sync = true
+  [
+    200,
+    {
+      'Content-Type' => 'text/plain',
+      'Transfer-Encoding' => 'chunked',
+    },
+    io
+  ].freeze
+}

data/t/async-response.ru

@@ -0,0 +1,13 @@
+use Rack::Chunked
+use Rainbows::DevFdResponse
+run lambda { |env|
+  io = IO.popen('for i in 0 1 2 3 4 5 6 7 8 9; do date; sleep 1; done', 'rb')
+  io.sync = true
+  [
+    200,
+    {
+      'Content-Type' => 'text/plain',
+    },
+    io
+  ].freeze
+}

data/t/bin/content-md5-put

@@ -22,7 +22,7 @@ if ARGV.grep("--no-headers").empty?
 end
 
 digest = Digest::MD5.new
-if buf = $stdin.read(bs)
+if buf = $stdin.readpartial(bs)
   begin
     digest.update(buf)
     $stdout.write("%x\r\n" % [ buf.size ])

data/t/bin/utee

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# tee(1) as distributed on most(all?) systems is buffered in luserspace
+# this only does unbuffered writes (with line-buffered input) to make
+# test output appear in real-time
+# -*- encoding: binary -*-
+$stdin.binmode
+$stdout.binmode
+fp = File.open(ARGV.shift, "wb")
+$stdin.each_line do |line|
+  fp.syswrite line
+  $stdout.syswrite line
+end

data/t/env.ru

@@ -0,0 +1,3 @@
+use Rack::ContentLength
+use Rack::ContentType
+run lambda { |env| [ 200, {}, [ env.inspect << "\n" ] ] }

data/t/large-file-response.ru

@@ -0,0 +1,13 @@
+# lib-large-file-response will stop running if we're not on Linux here
+use Rack::ContentLength
+use Rack::ContentType
+map "/rss" do
+  run lambda { |env|
+    # on Linux, this is in kilobytes
+    ::File.read("/proc/self/status") =~ /^VmRSS:\s+(\d+)/
+    [ 200, {}, [ ($1.to_i * 1024).to_s ] ]
+  }
+end
+map "/" do
+  run Rack::File.new(Dir.pwd)
+end

data/t/lib-async-response-no-autochunk.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+CONFIG_RU=async-response-no-autochunk.ru
+. ./lib-async-response.sh
+test x"$(cat $a)" = x0123456789
+test x"$(cat $b)" = x0123456789
+test x"$(cat $c)" = x0123456789

data/t/lib-async-response.sh

@@ -0,0 +1,45 @@
+CONFIG_RU=${CONFIG_RU-'async-response.ru'}
+. ./test-lib.sh
+echo "async response for model=$model"
+eval $(unused_listen)
+rtmpfiles unicorn_config a b c r_err r_out pid curl_err
+
+cat > $unicorn_config <<EOF
+listen "$listen"
+stderr_path "$r_err"
+stdout_path "$r_out"
+pid "$pid"
+Rainbows! { use :$model }
+EOF
+
+# can't load Rack::Lint here since it'll cause Rev to slurp
+rainbows -E none -D $CONFIG_RU -c $unicorn_config
+wait_for_pid $pid
+
+t0=$(date +%s)
+( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $a) &
+( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $b) &
+( curl --no-buffer -sSf http://$listen/ 2>> $curl_err | utee $c) &
+wait
+t1=$(date +%s)
+
+rainbows_pid=$(cat $pid)
+kill -QUIT $rainbows_pid
+elapsed=$(( $t1 - $t0 ))
+echo "elapsed=$elapsed < 30"
+test $elapsed -lt 30
+
+dbgcat a
+dbgcat b
+dbgcat c
+dbgcat r_err
+dbgcat curl_err
+test ! -s $curl_err
+check_stderr
+
+while kill -0 $rainbows_pid >/dev/null 2>&1
+do
+	sleep 1
+done
+
+dbgcat r_err

data/t/lib-graceful.sh

@@ -0,0 +1,40 @@
+. ./test-lib.sh
+echo "graceful test for model=$model"
+
+eval $(unused_listen)
+rtmpfiles unicorn_config curl_out pid r_err r_out fifo
+
+cat > $unicorn_config <<EOF
+listen "$listen"
+stderr_path "$r_err"
+stdout_path "$r_out"
+pid "$pid"
+Rainbows! { use :$model }
+EOF
+
+rainbows -D sleep.ru -c $unicorn_config
+wait_for_pid $pid
+rainbows_pid=$(cat $pid)
+
+curl -sSfv -T- </dev/null http://$listen/5 > $curl_out 2> $fifo &
+
+awk -v rainbows_pid=$rainbows_pid '
+{ print $0 }
+/100 Continue/ {
+	print "awk: sending SIGQUIT to", rainbows_pid
+	system("kill -QUIT "rainbows_pid)
+}' $fifo
+wait
+
+dbgcat r_err
+
+test x"$(wc -l < $curl_out)" = x1
+nr=$(sort < $curl_out | uniq | wc -l)
+
+test "$nr" -eq 1
+test x$(sort < $curl_out | uniq) = xHello
+check_stderr
+while kill -0 $rainbows_pid >/dev/null 2>&1
+do
+	sleep 1
+done

data/t/lib-input-trailer.sh

@@ -0,0 +1,63 @@
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+echo "input trailer test model=$model"
+
+eval $(unused_listen)
+rtmpfiles unicorn_config tmp r_err r_out pid fifo ok
+
+cat > $unicorn_config <<EOF
+listen "$listen"
+pid "$pid"
+stderr_path "$r_err"
+stdout_path "$r_out"
+Rainbows! { use :$model }
+EOF
+
+rainbows -D content-md5.ru -c $unicorn_config
+wait_for_pid $pid
+
+echo "small blob"
+(
+	echo hello world | content-md5-put
+	cat $fifo > $tmp &
+	wait
+	echo ok > $ok
+) | socat - TCP:$listen | utee $fifo
+
+fgrep 'HTTP/1.1 200 OK' $tmp
+test xok = x"$(cat $ok)"
+check_stderr
+
+echo "big blob"
+(
+	content-md5-put < random_blob
+	cat $fifo > $tmp &
+	wait
+	echo ok > $ok
+) | socat - TCP:$listen | utee $fifo
+
+fgrep 'HTTP/1.1 200 OK' $tmp
+test xok = x"$(cat $ok)"
+check_stderr
+
+echo "staggered blob"
+(
+	(
+		dd bs=164 count=1 < random_blob
+		sleep 2
+		dd bs=4545 count=1 < random_blob
+		sleep 2
+		dd bs=1234 count=1 < random_blob
+		echo ok > $ok
+	) 2>/dev/null | content-md5-put
+	test xok = x"$(cat $ok)"
+	cat $fifo > $tmp &
+	wait
+	echo ok > $ok
+) | socat - TCP:$listen | utee $fifo
+
+fgrep 'HTTP/1.1 200 OK' $tmp
+test xok = x"$(cat $ok)"
+check_stderr
+
+kill $(cat $pid)