rainbows 0.4.0 → 0.5.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.4.0.GIT
+DEF_VER=v0.5.0.GIT
 
 LF='
 '

data/GNUmakefile

@@ -64,7 +64,7 @@ NEWS: GIT-VERSION-FILE
 	$(rake) -s news_rdoc > $@+
 	mv $@+ $@
 
-SINCE = 0.1.1
+SINCE = 0.4.0
 ChangeLog: log_range = $(shell test -n "$(SINCE)" && echo v$(SINCE)..)
 ChangeLog: GIT-VERSION-FILE
 	@echo "ChangeLog from $(GIT_URL) ($(SINCE)..$(GIT_VERSION))" > $@+

data/bin/rainbows

@@ -4,7 +4,7 @@ require 'unicorn/launcher'
 require 'rainbows'
 require 'optparse'
 
-env = "development"
+ENV["RACK_ENV"] ||= "development"
 daemonize = false
 listeners = []
 options = { :listeners => listeners }
@@ -59,7 +59,7 @@ opts = OptionParser.new("", 24, '  ') do |opts|
 
   opts.on("-E", "--env ENVIRONMENT",
           "use ENVIRONMENT for defaults (default: development)") do |e|
-    env = e
+    ENV["RACK_ENV"] = e
   end
 
   opts.on("-D", "--daemonize", "run daemonized in the background") do |d|
@@ -134,7 +134,7 @@ app = lambda do ||
     Object.const_get(File.basename(config, '.rb').capitalize)
   end
   pp({ :inner_app => inner_app }) if $DEBUG
-  case env
+  case ENV["RACK_ENV"]
   when "development"
     Rack::Builder.new do
       use Rack::CommonLogger, $stderr

data/lib/rainbows/const.rb

@@ -3,7 +3,7 @@
 module Rainbows
 
   module Const
-    RAINBOWS_VERSION = '0.4.0'
+    RAINBOWS_VERSION = '0.5.0'
 
     include Unicorn::Const
 

data/lib/rainbows/ev_core.rb

@@ -14,7 +14,6 @@ module Rainbows
       @hp = HttpParser.new
       @state = :headers # [ :body [ :trailers ] ] :app_call :close
       @buf = ""
-      @deferred_bodies = [] # for (fast) regular files only
     end
 
     # graceful exit, like SIGQUIT
@@ -34,16 +33,8 @@ module Rainbows
         ERROR_500_RESPONSE
       end
       write(msg)
-      quit
-    end
-
-    def tmpio
-      io = Util.tmpio
-      def io.size
-        # already sync=true at creation, so no need to flush before stat
-        stat.size
-      end
-      io
+      ensure
+        quit
     end
 
     # TeeInput doesn't map too well to this right now...
@@ -63,7 +54,7 @@ module Rainbows
             write(EXPECT_100_RESPONSE)
             @env.delete(HTTP_EXPECT)
           end
-          @input = len && len <= MAX_BODY ? StringIO.new("") : tmpio
+          @input = len && len <= MAX_BODY ? StringIO.new("") : Util.tmpio
           @hp.filter_body(@buf2 = @buf.dup, @buf)
           @input << @buf2
           on_read("")
@@ -78,7 +69,10 @@ module Rainbows
           on_read("")
         end
       when :trailers
-        @hp.trailers(@env, @buf << data) and app_call
+        if @hp.trailers(@env, @buf << data)
+          app_call
+          @input.close if File === @input
+        end
       end
       rescue Object => e
         handle_error(e)

data/lib/rainbows/event_machine.rb

@@ -114,7 +114,7 @@ module Rainbows
           if do_chunk
             EM.watch(io, ResponseChunkPipe, self).notify_readable = true
           else
-            EM.enable_proxy(EM.attach(io, ResponsePipe, self), self)
+            EM.enable_proxy(EM.attach(io, ResponsePipe, self), self, 16384)
           end
         else
           HttpResponse.write(self, response, out)

data/lib/rainbows/http_server.rb

@@ -20,6 +20,21 @@ module Rainbows
       @worker_connections ||= MODEL_WORKER_CONNECTIONS[@use]
     end
 
+    #:stopdoc:
+    #
+    # Add one second to the timeout since our fchmod heartbeat is less
+    # precise (and must be more conservative) than Unicorn does.  We
+    # handle many clients per process and can't chmod on every
+    # connection we accept without wasting cycles.  That added to the
+    # fact that we let clients keep idle connections open for long
+    # periods of time means we have to chmod at a fixed interval.
+    alias_method :set_timeout, :timeout=
+    undef_method :timeout=
+    def timeout=(nr)
+      set_timeout(nr + 1)
+    end
+    #:startdoc:
+
     def use(*args)
       model = args.shift or return @use
       mod = begin
@@ -31,9 +46,9 @@ module Rainbows
       Module === mod or
         raise ArgumentError, "concurrency model #{model.inspect} not supported"
       extend(mod)
-      Const::RACK_DEFAULTS['rainbows.model'] = @use = model
+      Const::RACK_DEFAULTS['rainbows.model'] = @use = model.to_sym
       Const::RACK_DEFAULTS['rack.multithread'] = !!(/Thread/ =~ model.to_s)
-      case model
+      case @use
       when :Rev, :EventMachine
         Const::RACK_DEFAULTS['rainbows.autochunk'] = true
       end

data/lib/rainbows/rev.rb

@@ -1,6 +1,5 @@
 # -*- encoding: binary -*-
-require 'rev'
-Rev::VERSION >= '0.3.0' or abort 'rev >= 0.3.0 is required'
+require 'rainbows/rev/heartbeat'
 require 'rainbows/ev_core'
 
 module Rainbows
@@ -35,6 +34,7 @@ module Rainbows
         G.cur += 1
         super(io)
         post_init
+        @deferred_bodies = [] # for (fast) regular files only
       end
 
       # queued, optional response bodies, it should only be unpollable "fast"
@@ -168,22 +168,6 @@ module Rainbows
       end
     end
 
-    # This timer handles the fchmod heartbeat to prevent our master
-    # from killing us.
-    class Heartbeat < ::Rev::TimerWatcher
-      G = Rainbows::G
-
-      def initialize(tmp)
-        @m, @tmp = 0, tmp
-        super(1, true)
-      end
-
-      def on_timer
-        @tmp.chmod(@m = 0 == @m ? 1 : 0)
-        exit if (! G.alive && G.cur <= 0)
-      end
-    end
-
     # runs inside each forked worker, this sits around and waits
     # for connections and doesn't die until the parent dies (or is
     # given a INT, QUIT, or TERM signal)

data/lib/rainbows/rev/heartbeat.rb

@@ -0,0 +1,27 @@
+# -*- encoding: binary -*-
+require 'rev'
+Rev::VERSION >= '0.3.0' or abort 'rev >= 0.3.0 is required'
+
+module Rainbows
+  module Rev
+
+    # This class handles the Unicorn fchmod heartbeat mechanism
+    # in Rev-based concurrency models to prevent the master
+    # process from killing us unless we're blocked.  This class
+    # will also detect and execute the graceful exit if triggered
+    # by SIGQUIT
+    class Heartbeat < ::Rev::TimerWatcher
+      # +tmp+ must be a +File+ that responds to +chmod+
+      def initialize(tmp)
+        @m, @tmp = 0, tmp
+        super(1, true)
+      end
+
+      def on_timer
+        @tmp.chmod(@m = 0 == @m ? 1 : 0)
+        exit if (! G.alive && G.cur <= 0)
+      end
+
+    end
+  end
+end

data/lib/rainbows/thread_pool.rb

@@ -34,7 +34,7 @@ module Rainbows
         pool.each do |thr|
           worker.tmp.chmod(m = 0 == m ? 1 : 0)
           # if any worker dies, something is serious wrong, bail
-          thr.join(timeout) and break
+          thr.join(1) and break
         end
       end
       join_threads(pool, worker)
@@ -44,6 +44,11 @@ module Rainbows
       Thread.new {
         begin
           begin
+            # TODO: check if select() or accept() is a problem on large
+            # SMP systems under Ruby 1.9.  Hundreds of native threads
+            # all working off the same socket could be a thundering herd
+            # problem.  On the other hand, a thundering herd may not
+            # even incur as much overhead as an extra Mutex#synchronize
             ret = IO.select(LISTENERS, nil, nil, 1) and
                   ret.first.each do |sock|
                     begin

data/local.mk.sample

@@ -6,7 +6,7 @@
 
 DLEXT := so
 gems := rack-1.0.1
-# gems += unicorn-0.93.3 # installed via setup.rb
+# gems += unicorn-0.93.5 # installed via setup.rb
 gems += rev-0.3.1 iobuffer-0.1.1
 gems += eventmachine-0.12.10
 gems += async_sinatra-0.1.5 sinatra-0.9.4

data/rainbows.gemspec

@@ -41,7 +41,7 @@ Gem::Specification.new do |s|
   s.test_files = test_files
 
   # we need Unicorn for the HTTP parser and process management
-  s.add_dependency(%q<unicorn>, ["~> 0.93.4"])
+  s.add_dependency(%q<unicorn>, ["~> 0.94.0"])
 
   # Unicorn already depends on Rack
   # s.add_dependency(%q<rack>)

data/t/GNUmakefile

@@ -82,7 +82,7 @@ $(deps): dep_bin = $(lastword $(subst +, ,$@))
 $(deps):
 	@which $(dep_bin) > $@.$(pid) 2>/dev/null || :
 	@test -s $@.$(pid) || \
-	  { echo >&2 "E `$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
+	  { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
 	@mv $@.$(pid) $@
 dep: $(deps)
 

data/t/env_rack_env.ru

@@ -0,0 +1,4 @@
+use Rack::ContentLength
+run proc { |env|
+  [ 200, { "Content-Type" => "text/plain" }, [ ENV["RACK_ENV"] ] ]
+}

data/t/heartbeat-timeout.ru

@@ -0,0 +1,14 @@
+use Rack::ContentLength
+fifo = ENV['FIFO_PATH'] or abort "FIFO_PATH not defined"
+headers = { 'Content-Type' => 'text/plain' }
+run lambda { |env|
+  case env['PATH_INFO']
+  when "/block-forever"
+    # one of these should block forever
+    Process.kill(:STOP, $$)
+    ::File.open(fifo, "rb") { |fp| fp.syswrite("NEVER\n") }
+    [ 500, headers, [ "Should never get here\n" ] ]
+  else
+    [ 200, headers, [ "#$$\n" ] ]
+  end
+}

data/t/t0004-heartbeat-timeout.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+. ./test-lib.sh
+
+t_plan 9 "heartbeat/timeout test for $model"
+
+t_begin "setup and startup" && {
+	rainbows_setup $model
+        echo timeout 3 >> $unicorn_config
+        echo preload_app true >> $unicorn_config
+	FIFO_PATH=$fifo rainbows -D heartbeat-timeout.ru -c $unicorn_config
+	rainbows_wait_start
+}
+
+t_begin "read worker PID" && {
+	worker_pid=$(curl -sSf http://$listen/)
+	t_info "worker_pid=$worker_pid"
+}
+
+t_begin "sleep for a bit, ensure worker PID does not change" && {
+	sleep 4
+	test $(curl -sSf http://$listen/) -eq $worker_pid
+}
+
+t_begin "block the worker process to force it to die" && {
+	t0=$(date +%s)
+	err="$(curl -sSf http://$listen/block-forever 2>&1 || :)"
+	t1=$(date +%s)
+	elapsed=$(($t1 - $t0))
+	t_info "elapsed=$elapsed err=$err"
+	test x"$err" != x"Should never get here"
+	test x"$err" != x"$worker_pid"
+}
+
+t_begin "ensure timeout took 3-6 seconds" && {
+	test $elapsed -ge 3
+	test $elapsed -le 6 # give it some slack in case box is bogged down
+}
+
+t_begin "wait for new worker to start up" && {
+	test x = x"$(cat $fifo)"
+}
+
+t_begin "we get a fresh new worker process" && {
+	new_worker_pid=$(curl -sSf http://$listen/)
+	test $new_worker_pid -ne $worker_pid
+}
+
+t_begin "SIGSTOP and SIGCONT on rainbows master does not kill worker" && {
+	kill -STOP $rainbows_pid
+	sleep 4
+	kill -CONT $rainbows_pid
+	sleep 2
+	test $new_worker_pid -eq $(curl -sSf http://$listen/)
+}
+
+t_begin "stop server" && {
+	kill $rainbows_pid
+}
+
+dbgcat r_err
+
+t_done

data/t/t0006-process-rack-env.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+. ./test-lib.sh
+
+t_plan 4 'ensure ENV["RACK_ENV"] is set correctly for '$model
+
+finish_checks () {
+	kill $rainbows_pid
+	test ! -s $curl_err
+	check_stderr
+}
+
+t_begin "setup" && {
+	rtmpfiles curl_out curl_err
+}
+
+t_begin "default RACK_ENV is 'development'" && {
+	rainbows_setup
+	rainbows -D -c $unicorn_config env_rack_env.ru
+	rainbows_wait_start
+	test x"$(curl -sSf http://$listen 2>$curl_err)" = x"development"
+	finish_checks
+}
+
+t_begin "RACK_ENV from process ENV is inherited" && {
+	rainbows_setup
+	( RACK_ENV=production rainbows -D -c $unicorn_config env_rack_env.ru )
+	rainbows_wait_start
+	test x$(curl -sSf http://$listen 2>$curl_err) = x"production"
+	finish_checks
+}
+
+t_begin "RACK_ENV from -E is set" && {
+	rainbows_setup
+	rainbows -D -c $unicorn_config -E none env_rack_env.ru
+	rainbows_wait_start
+	test x$(curl -sSf http://$listen 2>$curl_err) = x"none"
+	finish_checks
+}
+
+t_done

data/t/t0100-rack-input-hammer.sh

@@ -35,7 +35,7 @@ t_begin "all responses identical" && {
 }
 
 t_begin "sha1 matches on-disk sha1" && {
-	blob_sha1=$( expr "$(sha1sum < random_blob)" : '\([a-f0-9]\+\)')
+	blob_sha1=$( expr "$(sha1sum < random_blob)" : '\([a-f0-9]\{40\}\)')
 	t_info blob_sha1=$blob_sha1
 	test x"$blob_sha1" = x"$(sort < $curl_out | uniq)"
 }

data/t/t0101-rack-input-trailer.sh

@@ -2,7 +2,7 @@
 . ./test-lib.sh
 test -r random_blob || die "random_blob required, run with 'make $0'"
 
-t_plan 11 "input trailer test $model"
+t_plan 13 "input trailer test $model"
 
 t_begin "setup and startup" && {
 	rtmpfiles curl_out
@@ -11,6 +11,27 @@ t_begin "setup and startup" && {
 	rainbows_wait_start
 }
 
+t_begin "staggered trailer upload" && {
+	zero_md5="1B2M2Y8AsgTpgAmY7PhCfg=="
+	(
+		cat $fifo > $tmp &
+		printf 'PUT /s HTTP/1.1\r\n'
+		printf 'Host: example.com\r\n'
+		printf 'Transfer-Encoding: chunked\r\n'
+		printf 'Trailer: Content-MD5\r\n\r\n'
+		printf '0\r\nContent-MD5: '
+		sleep 5
+		printf '%s\r\n\r\n' $zero_md5
+		wait
+		echo ok > $ok
+	) | socat - TCP:$listen > $fifo
+	test xok = x"$(cat $ok)"
+}
+
+t_begin "HTTP response is OK" && {
+	fgrep 'HTTP/1.1 200 OK' $tmp
+}
+
 t_begin "upload small blob" && {
 	(
 		cat $fifo > $tmp &

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rainbows
 version: !ruby/object:Gem::Version 
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Rainbows! developers
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-27 00:00:00 -07:00
+date: 2009-11-05 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.93.4
+        version: 0.94.0
     version: 
 description: |-
   Rainbows! is a HTTP server for sleepy Rack applications.  It is based on
@@ -47,6 +47,7 @@ extra_rdoc_files:
 - lib/rainbows/http_response.rb
 - lib/rainbows/http_server.rb
 - lib/rainbows/rev.rb
+- lib/rainbows/rev/heartbeat.rb
 - lib/rainbows/revactor.rb
 - lib/rainbows/revactor/tee_input.rb
 - lib/rainbows/thread_pool.rb
@@ -90,6 +91,7 @@ files:
 - lib/rainbows/http_response.rb
 - lib/rainbows/http_server.rb
 - lib/rainbows/rev.rb
+- lib/rainbows/rev/heartbeat.rb
 - lib/rainbows/revactor.rb
 - lib/rainbows/revactor/tee_input.rb
 - lib/rainbows/thread_pool.rb
@@ -109,6 +111,8 @@ files:
 - t/bin/utee
 - t/content-md5.ru
 - t/env.ru
+- t/env_rack_env.ru
+- t/heartbeat-timeout.ru
 - t/large-file-response.ru
 - t/my-tap-lib.sh
 - t/sha1.ru
@@ -125,16 +129,14 @@ files:
 - t/t0002-graceful.sh
 - t/t0002-parser-error.sh
 - t/t0003-reopen-logs.sh
+- t/t0004-heartbeat-timeout.sh
 - t/t0005-large-file-response.sh
+- t/t0006-process-rack-env.sh
 - t/t0100-rack-input-hammer.sh
 - t/t0101-rack-input-trailer.sh
 - t/t0200-async-response.sh
 - t/t0201-async-response-no-autochunk.sh
 - t/t0300-async_sinatra.sh
-- t/t1000.ru
-- t/t2000.ru
-- t/t3000.ru
-- t/t4000.ru
 - t/t9000-rack-app-pool.sh
 - t/t9000.ru
 - t/test-lib.sh

data/t/t1000.ru

@@ -1,10 +0,0 @@
-use Rack::ContentLength
-use Rack::ContentType
-run lambda { |env|
-  sleep 1
-  if env['rack.multithread'] && env['rainbows.model'] == :ThreadPool
-    [ 200, {}, [ Thread.current.inspect << "\n" ] ]
-  else
-    raise "rack.multithread is not true"
-  end
-}

data/t/t2000.ru

@@ -1,10 +0,0 @@
-use Rack::ContentLength
-use Rack::ContentType
-run lambda { |env|
-  sleep 1
-  if env['rack.multithread'] && env['rainbows.model'] == :ThreadSpawn
-    [ 200, {}, [ Thread.current.inspect << "\n" ] ]
-  else
-    raise "rack.multithread is not true"
-  end
-}

data/t/t3000.ru

@@ -1,10 +0,0 @@
-use Rack::ContentLength
-use Rack::ContentType
-run lambda { |env|
-  Actor.sleep 1
-  if env['rack.multithread'] == false && env['rainbows.model'] == :Revactor
-    [ 200, {}, [ Thread.current.inspect << "\n" ] ]
-  else
-    raise "rack.multithread is true"
-  end
-}

data/t/t4000.ru

@@ -1,9 +0,0 @@
-use Rack::ContentLength
-use Rack::ContentType
-run lambda { |env|
-  if env['rack.multithread'] == false && env['rainbows.model'] == :Rev
-    [ 200, {}, [ env.inspect << "\n" ] ]
-  else
-    raise "rack.multithread is true"
-  end
-}