rainbows 0.94.0 → 0.95.0

data/lib/rainbows.rb

@@ -1,6 +1,10 @@
 # -*- encoding: binary -*-
 require 'unicorn'
+# the value passed to TCP_DEFER_ACCEPT actually matters in Linux 2.6.32+
+Unicorn::SocketHelper::DEFAULTS[:tcp_defer_accept] = 60
+
 require 'rainbows/error'
+require 'rainbows/configurator'
 require 'fcntl'
 
 module Rainbows
@@ -8,6 +12,7 @@ module Rainbows
   # global vars because class/instance variables are confusing me :<
   # this struct is only accessed inside workers and thus private to each
   # G.cur may not be used in the network concurrency model
+  # :stopdoc:
   class State < Struct.new(:alive,:m,:cur,:kato,:server,:tmp,:expire)
     def tick
       tmp.chmod(self.m = m == 0 ? 1 : 0)
@@ -22,14 +27,13 @@ module Rainbows
       false
     end
   end
-  # :stopdoc:
   G = State.new(true, 0, 0, 5)
   O = {}
   # :startdoc:
 
   require 'rainbows/const'
   require 'rainbows/http_server'
-  require 'rainbows/http_response'
+  require 'rainbows/response'
   require 'rainbows/base'
   require 'rainbows/tee_input'
   autoload :Sendfile, 'rainbows/sendfile'
@@ -60,12 +64,12 @@ module Rainbows
 
     # runs the Rainbows! HttpServer with +app+ and +options+ and does
     # not return until the server has exited.
-    def run(app, options = {})
+    def run(app, options = {}) # :nodoc:
       HttpServer.new(app, options).start.join
     end
 
     # returns nil if accept fails
-    def sync_accept(sock)
+    def sync_accept(sock) # :nodoc:
       rv = sock.accept
       rv.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
       rv
@@ -73,7 +77,7 @@ module Rainbows
     end
 
     # returns nil if accept fails
-    def accept(sock)
+    def accept(sock) # :nodoc:
       rv = sock.accept_nonblock
       rv.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
       rv
@@ -83,50 +87,18 @@ module Rainbows
     # returns a string representing the address of the given client +io+
     # For local UNIX domain sockets, this will return a string referred
     # to by the (non-frozen) Unicorn::HttpRequest::LOCALHOST constant.
-    def addr(io)
+    def addr(io) # :nodoc:
       io.respond_to?(:peeraddr) ?
-                        io.peeraddr.last : Unicorn::HttpRequest::LOCALHOST
+                        io.peeraddr[-1] : Unicorn::HttpRequest::LOCALHOST
     end
 
+    # :stopdoc:
     # the default max body size is 1 megabyte (1024 * 1024 bytes)
     @@max_bytes = 1024 * 1024
 
     def max_bytes; @@max_bytes; end
     def max_bytes=(nr); @@max_bytes = nr; end
-  end
-
-  # configures \Rainbows! with a given concurrency model to +use+ and
-  # a +worker_connections+ upper-bound.  This method may be called
-  # inside a Unicorn/Rainbows configuration file:
-  #
-  #   Rainbows! do
-  #     use :Revactor # this may also be :ThreadSpawn or :ThreadPool
-  #     worker_connections 400
-  #     keepalive_timeout 0 # zero disables keepalives entirely
-  #     client_max_body_size 5*1024*1024 # 5 megabytes
-  #   end
-  #
-  #   # the rest of the Unicorn configuration
-  #   worker_processes 8
-  #
-  # See the documentation for the respective Revactor, ThreadSpawn,
-  # and ThreadPool classes for descriptions and recommendations for
-  # each of them.  The total number of clients we're able to serve is
-  # +worker_processes+ * +worker_connections+, so in the above example
-  # we can serve 8 * 400 = 3200 clients concurrently.
-  #
-  # The default is +keepalive_timeout+ is 5 seconds, which should be
-  # enough under most conditions for browsers to render the page and
-  # start retrieving extra elements for.  Increasing this beyond 5
-  # seconds is not recommended.  Zero disables keepalive entirely
-  # (but pipelining fully-formed requests is still works).
-  #
-  # The default +client_max_body_size+ is 1 megabyte (1024 * 1024 bytes),
-  # setting this to +nil+ will disable body size checks and allow any
-  # size to be specified.
-  def Rainbows!(&block)
-    block_given? or raise ArgumentError, "Rainbows! requires a block"
-    HttpServer.setup(block)
+    # :startdoc:
   end
 
   # :stopdoc:
@@ -155,8 +127,7 @@ module Rainbows
   end
   # :startdoc:
   autoload :Fiber, 'rainbows/fiber' # core class
-
+  autoload :ByteSlice, 'rainbows/byte_slice'
+  autoload :StreamFile, 'rainbows/stream_file'
+  autoload :HttpResponse, 'rainbows/http_response' # deprecated
 end
-
-# inject the Rainbows! method into Unicorn::Configurator
-Unicorn::Configurator.class_eval { include Rainbows }

data/rainbows.gemspec

@@ -40,17 +40,17 @@ Gem::Specification.new do |s|
 
   s.test_files = test_files
 
-  # we need Unicorn for the HTTP parser and process management
-  # The HTTP parser in Unicorn <= 0.97.0 was vulnerable to a remote DoS
-  # when exposed directly to untrusted clients.
-  s.add_dependency(%q<unicorn>, [">= 0.97.1", "< 2.0.0"])
-  s.add_development_dependency(%q<isolate>, "~> 2.0.2")
+  # we want a newer Rack for a valid HeaderHash#each
+  s.add_dependency(%q<rack>, ['~> 1.1'])
 
-  # Unicorn already depends on Rack
-  # s.add_dependency(%q<rack>)
+  # we need Unicorn for the HTTP parser and process management
+  # Unicorn 0.991.0 handles config.ru when started outside of
+  # the prespecified working_directory
+  s.add_dependency(%q<unicorn>, [">= 1.1.0", "< 2.0.0"])
+  s.add_development_dependency(%q<isolate>, "~> 2.1.0")
 
   # optional runtime dependencies depending on configuration
-  # see config/isolate.rb for the exact versions we've tested with
+  # see t/test_isolate.rb for the exact versions we've tested with
   #
   # Revactor >= 0.1.5 includes UNIX domain socket support
   # s.add_dependency(%q<revactor>, [">= 0.1.5"])

data/t/.gitignore

@@ -1,5 +1,5 @@
 /test-results-*
-/test-bin-*
+/bin-*
 /random_blob
 /.dep+*
 /trash

data/t/GNUmakefile

@@ -4,6 +4,7 @@ all::
 
 pid := $(shell echo $$PPID)
 
+MRI = ruby
 RUBY = ruby
 rainbows_lib := $(shell cd ../lib && pwd)
 -include ../local.mk
@@ -15,7 +16,8 @@ ifeq ($(RUBY_VERSION),)
   $(error unable to detect RUBY_VERSION)
 endif
 
-export RUBY_VERSION
+RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
+export RUBY_VERSION RUBY_ENGINE
 
 models += WriterThreadPool
 models += WriterThreadSpawn
@@ -27,13 +29,19 @@ models += NeverBlock
 models += RevThreadSpawn
 models += RevThreadPool
 
-rp := )
-ONENINE := $(shell case $(RUBY_VERSION) in 1.9.*$(rp) echo true;;esac)
-ifeq ($(ONENINE),true)
-  models += Revactor
-  models += FiberSpawn
-  models += RevFiberSpawn
-  models += FiberPool
+ifeq ($(RUBY_ENGINE),ruby)
+  rp := )
+  ONENINE := $(shell case $(RUBY_VERSION) in 1.9.*$(rp) echo true;;esac)
+  ifeq ($(ONENINE),true)
+    models += Revactor
+    models += FiberSpawn
+    models += RevFiberSpawn
+    models += FiberPool
+  endif
+endif
+
+ifeq ($(RUBY_ENGINE),rbx)
+  models += ActorSpawn
 endif
 all_models := $(models) Base
 
@@ -60,7 +68,7 @@ $(all_models):
 all:: $(T)
 
 # can't rely on "set -o pipefail" since we don't require bash or ksh93 :<
-t_pfx = trash/$@-$(RUBY_VERSION)
+t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION)
 TEST_OPTS =
 # TRACER = strace -f -o $(t_pfx).strace -s 100000
 # TRACER = /usr/bin/time -o $(t_pfx).time
@@ -73,11 +81,13 @@ ifdef V
   endif
 endif
 
-test-bin-$(RUBY_VERSION)/rainbows: ruby_bin = $(shell which $(RUBY))
-test-bin-$(RUBY_VERSION)/rainbows: ../bin/rainbows
+bindir := $(CURDIR)/bin-$(RUBY_ENGINE)-$(RUBY_VERSION)
+bin_rainbows := $(bindir)/rainbows
+$(bin_rainbows): ruby_bin = $(shell which $(RUBY))
+$(bin_rainbows): ../bin/rainbows
 	mkdir -p $(@D)
 	install -m 755 $^ $@.$(pid)
-	$(RUBY) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@.$(pid)
+	$(MRI) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@.$(pid)
 	mv $@.$(pid) $@
 
 random_blob:
@@ -95,18 +105,18 @@ $(deps):
 	  { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
 	@mv $@.$(pid) $@
 
-libs := tmp/isolate/ruby-$(RUBY_VERSION)/.libs
+libs := tmp/isolate/$(RUBY_ENGINE)-$(RUBY_VERSION)/.libs
 $(libs): test_isolate.rb
 	mkdir -p $(@D)
 	$(RUBY) $< > $@+
 	mv $@+ $@
-t_deps := $(libs) $(deps) test-bin-$(RUBY_VERSION)/rainbows trash/.gitignore
+t_deps := $(libs) $(deps) $(bin_rainbows) trash/.gitignore
 $(T): $(t_deps)
 
 $(MODEL_T): export model = $(firstword $(subst ., ,$@))
 $(MODEL_T): script = $(subst $(model).,,$@)
 $(MODEL_T): export RUBY := $(RUBY)
-$(MODEL_T): export PATH := $(CURDIR)/test-bin-$(RUBY_VERSION):$(PATH)
+$(MODEL_T): export PATH := $(bindir):$(PATH)
 $(MODEL_T): $(t_deps)
 	RUBYLIB=$(rainbows_lib):$$(cat $(libs)) \
 	   $(TRACER) $(SHELL) $(SH_TEST_OPTS) $(script) $(TEST_OPTS)
@@ -116,6 +126,6 @@ trash/.gitignore:
 	echo '*' > $@
 
 clean:
-	$(RM) -r trash/*.log trash/*.code test-bin-$(RUBY_VERSION)
+	$(RM) -r trash/*.log trash/*.code $(bindir)
 
 .PHONY: $(T) clean

data/t/README

@@ -12,7 +12,7 @@ easily portable to non-Ruby web servers.
 == Requirements
 
 * {Ruby 1.8 or 1.9}[http://www.ruby-lang.org/] (duh!)
-* {isolate ~> 2.0.2}[http://github.com/jbarnette/isolate] - for dependencies
+* {isolate ~> 2.1.0}[http://github.com/jbarnette/isolate] - for dependencies
 * {GNU make}[http://www.gnu.org/software/make/]
 * {socat}[http://www.dest-unreach.org/socat/]
 * {curl >= 7.18.0}[http://curl.haxx.se/]

data/t/async-response-no-autochunk.ru

@@ -12,7 +12,6 @@ EOF
 run lambda { |env|
   env['rainbows.autochunk'] = false
   io = IO.popen(script, 'rb')
-  io.sync = true
   [
     200,
     {

data/t/async-response.ru

@@ -2,7 +2,6 @@ use Rack::Chunked
 use Rainbows::DevFdResponse
 run lambda { |env|
   io = IO.popen('for i in 0 1 2 3 4 5 6 7 8 9; do date; sleep 1; done', 'rb')
-  io.sync = true
   [
     200,
     {

data/t/cramp/rainsocket.ru

@@ -0,0 +1,26 @@
+# based on examples/rainsocket.ru git://github.com/lifo/cramp
+# Rack::Lint does not like async + EM stuff, so disable it:
+#\ -E deployment
+require 'cramp/controller'
+
+Cramp::Controller::Websocket.backend = :rainbows
+
+class WelcomeController < Cramp::Controller::Websocket
+  periodic_timer :send_hello_world, :every => 2
+  on_data :received_data
+
+  def received_data(data)
+    if data =~ /fuck/
+      render "You cant say fuck in here"
+      finish
+    else
+      render "Got your #{data}"
+    end
+  end
+
+  def send_hello_world
+    render "Hello from the Server!\n"
+  end
+end
+
+run WelcomeController

data/t/fork-sleep.ru

@@ -1,4 +1,3 @@
-#\-E none
 # we do not want Rack::Lint or anything to protect us
 use Rack::ContentLength
 use Rack::ContentType, "text/plain"

data/t/my-tap-lib.sh

@@ -188,9 +188,10 @@ then
 
 	(
 		# use a subshell so seds are not waitable
-		$SED -e 's/^/#: /' $t_stdout &
-		$SED -e 's/^/#! /' $t_stderr &
+		$SED -e 's/^/#: /' < $t_stdout &
+		$SED -e 's/^/#! /' < $t_stderr &
 	) &
+	wait
 	exec > $t_stdout 2> $t_stderr
 else
 	exec > /dev/null 2> /dev/null

data/t/simple-http_ActorSpawn.ru

@@ -0,0 +1,9 @@
+use Rack::ContentLength
+use Rack::ContentType
+run lambda { |env|
+  if env['rack.multithread'] && env['rainbows.model'] == :ActorSpawn
+    [ 200, {}, [ Actor.current.inspect << "\n" ] ]
+  else
+    raise "rack.multithread is not true"
+  end
+}

data/t/t0009-broken-app.sh

@@ -5,7 +5,7 @@ t_plan 9 "graceful handling of broken apps for $model"
 
 t_begin "setup and start" && {
 	rainbows_setup $model 1
-	rainbows -D t0009.ru -c $unicorn_config
+	rainbows -E none -D t0009.ru -c $unicorn_config
 	rainbows_wait_start
 }
 

data/t/t0009.ru

@@ -1,4 +1,3 @@
-#\-E none
 # we do not want Rack::Lint or anything to protect us
 use Rack::ContentLength
 use Rack::ContentType, "text/plain"

data/t/t0011-close-on-exec-set.sh

@@ -6,7 +6,7 @@ t_plan 7 "ensure close-on-exec flag is set for $model"
 
 t_begin "setup and start" && {
 	rainbows_setup $model 1 1
-	nr=$nr rainbows -D fork-sleep.ru -c $unicorn_config
+	nr=$nr rainbows -E none -D fork-sleep.ru -c $unicorn_config
 	rainbows_wait_start
 }
 

data/t/t0015-working_directory.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+if test -n "$RBX_SKIP"
+then
+	echo "$0 is broken under Rubinius for now"
+	exit 0
+fi
+. ./test-lib.sh
+
+t_plan 4 "config.ru inside alt working_directory"
+
+t_begin "setup and start" && {
+	rainbows_setup
+	rtmpfiles unicorn_config_tmp
+	rm -rf $t_pfx.app
+	mkdir $t_pfx.app
+
+	cat > $t_pfx.app/config.ru <<EOF
+#\--daemonize --listen $listen
+use Rack::ContentLength
+use Rack::ContentType, "text/plain"
+run lambda { |env| [ 200, {}, [ "#{\$master_ppid}\\n" ] ] }
+EOF
+	# we have --host/--port in config.ru instead
+	grep -v ^listen $unicorn_config > $unicorn_config_tmp
+
+	# the whole point of this exercise
+	echo "working_directory '$t_pfx.app'" >> $unicorn_config_tmp
+
+	# allows ppid to be 1 in before_fork
+	echo "preload_app true" >> $unicorn_config_tmp
+	cat >> $unicorn_config_tmp <<\EOF
+before_fork do |server,worker|
+  $master_ppid = Process.ppid # should be zero to detect daemonization
+end
+EOF
+
+	mv $unicorn_config_tmp $unicorn_config
+
+	# rely on --daemonize switch, no & or -D
+	rainbows -c $unicorn_config
+	rainbows_wait_start
+}
+
+t_begin "hit with curl" && {
+	body=$(curl -sSf http://$listen/)
+}
+
+t_begin "killing succeeds" && {
+	kill $rainbows_pid
+}
+
+t_begin "response body ppid == 1 (daemonized)" && {
+	test "$body" -eq 1
+}
+
+t_done

data/t/t0016-onenine-encoding-is-tricky.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 4 "proper handling of onenine encoding for $model"
+
+t_begin "setup and startup" && {
+	rainbows_setup $model
+	rainbows -D ./t0016.rb -c $unicorn_config
+	rainbows_wait_start
+	expect_sha1=8ff79d8115f9fe38d18be858c66aa08a1cc27a66
+}
+
+t_begin "response matches expected" && {
+	rm -f $ok
+	(
+		curl -sSf http://$listen/ && echo ok > $ok
+	) | rsha1 > $tmp
+	test x$expect_sha1 = x"$(cat $tmp)"
+}
+
+t_begin "shutdown server" && {
+	kill -QUIT $rainbows_pid
+}
+
+dbgcat r_err
+
+t_begin "check stderr" && check_stderr
+
+t_done

data/t/t0016.rb

@@ -0,0 +1,15 @@
+# -*- encoding: utf-8 -*-
+module T0016
+  CHUNK = '©' * 1024 * 1024
+  BODY = (1..50).map { CHUNK }
+  HEADER = {
+    # BODY.inject(0) { |m,c| m += c.bytesize }.to_s,
+    'Content-Length' => '104857600',
+    'Content-Type' => 'text/plain',
+  }
+
+  def self.call(env)
+    [ 200, HEADER, BODY ]
+  end
+end
+$0 == __FILE__ and T0016::BODY.each { |x| $stdout.syswrite(x) }

data/t/t0020-large-sendfile-response.sh

@@ -0,0 +1,141 @@
+#!/bin/sh
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+case $RUBY_ENGINE in
+ruby) ;;
+*)
+	t_info "skipping $T since it can't load the sendfile gem, yet"
+	exit 0
+	;;
+esac
+
+t_plan 12 "large sendfile response for $model"
+
+t_begin "setup and startup" && {
+	rtmpfiles curl_out a b c slow_a slow_b
+	rainbows_setup $model
+	echo 'require "sendfile"' >> $unicorn_config
+	echo 'def (::IO).copy_stream(*x); abort "NO"; end' >> $unicorn_config
+
+	# can't load Rack::Lint here since it clobbers body#to_path
+	rainbows -E none -D large-file-response.ru -c $unicorn_config
+	rainbows_wait_start
+}
+
+t_begin "read random blob sha1" && {
+	random_blob_sha1=$(rsha1 < random_blob)
+	three_sha1=$(cat random_blob random_blob random_blob | rsha1)
+}
+
+t_begin "send keepalive HTTP/1.1 requests in parallel" && {
+	for i in $a $b $c $slow_a $slow_b
+	do
+		curl -sSf http://$listen/random_blob \
+		          http://$listen/random_blob \
+		          http://$listen/random_blob | rsha1 > $i &
+	done
+	wait
+	for i in $a $b $c $slow_a $slow_b
+	do
+		test x$(cat $i) = x$three_sha1
+	done
+}
+
+t_begin "send a batch of abortive HTTP/1.1 requests in parallel" && {
+	for i in $a $b $c $slow_a $slow_b
+	do
+		rm -f $i
+		(
+			curl -sSf --max-time 5 --limit-rate 1K \
+			  http://$listen/random_blob >/dev/null || echo ok > $i
+		) &
+	done
+	wait
+}
+
+t_begin "all requests timed out" && {
+	for i in $a $b $c $slow_a $slow_b
+	do
+		test x$(cat $i) = xok
+	done
+}
+
+s='$NF ~ /worker_connections=[0-9]+/{gsub(/[^0-9]/,"",$3); print $3; exit}'
+t_begin "check proc to ensure file is closed properly (Linux only)" && {
+	worker_pid=$(awk "$s" < $r_err)
+	test -n "$worker_pid"
+	if test -d /proc/$worker_pid/fd
+	then
+		if ls -l /proc/$worker_pid/fd | grep random_blob
+		then
+			t_info "random_blob file is open ($model)"
+		fi
+	else
+		t_info "/proc/$worker_pid/fd not found"
+	fi
+}
+
+t_begin "send a bunch of HTTP/1.1 requests in parallel" && {
+	(
+		curl -sSf --limit-rate 1M http://$listen/random_blob | \
+		  rsha1 > $slow_a
+	) &
+	(
+		curl -sSf --limit-rate 750K http://$listen/random_blob | \
+		  rsha1 > $slow_b
+	) &
+	for i in $a $b $c
+	do
+		(
+			curl -sSf http://$listen/random_blob | rsha1 > $i
+		) &
+	done
+	wait
+	for i in $a $b $c $slow_a $slow_b
+	do
+		test x$(cat $i) = x$random_blob_sha1
+	done
+}
+
+# this was a problem during development
+t_begin "HTTP/1.0 test" && {
+	sha1=$( (curl -0 -sSf http://$listen/random_blob &&
+	         echo ok >$ok) | rsha1)
+	test $sha1 = $random_blob_sha1
+	test xok = x$(cat $ok)
+}
+
+t_begin "HTTP/0.9 test" && {
+	(
+		printf 'GET /random_blob\r\n'
+		rsha1 < $fifo > $tmp &
+		wait
+		echo ok > $ok
+	) | socat - TCP:$listen > $fifo
+	test $(cat $tmp) = $random_blob_sha1
+	test xok = x$(cat $ok)
+}
+
+t_begin "check proc to ensure file is closed properly (Linux only)" && {
+	worker_pid=$(awk "$s" < $r_err)
+	test -n "$worker_pid"
+	if test -d /proc/$worker_pid/fd
+	then
+		if ls -l /proc/$worker_pid/fd | grep random_blob
+		then
+			t_info "random_blob file is open ($model)"
+		fi
+	else
+		t_info "/proc/$worker_pid/fd not found"
+	fi
+}
+
+t_begin "shutdown server" && {
+	kill -QUIT $rainbows_pid
+}
+
+dbgcat r_err
+
+t_begin "check stderr" && check_stderr
+
+t_done

data/t/t0300-async_sinatra.sh

@@ -1,11 +1,5 @@
 #!/bin/sh
 . ./test-lib.sh
-case $RUBY_VERSION in
-1.9.2)
-	t_info "RUBY_VERSION=$RUBY_VERSION not supported with async_sinatra"
-	exit 0
-	;;
-esac
 
 # n - number of seconds to sleep
 n=10

data/t/t0501-cramp-rainsocket.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+. ./test-lib.sh
+case $model in
+EventMachine) ;;
+*)
+	t_info "skipping $T since it's not compatible with $model"
+	exit 0
+	;;
+esac
+require_check cramp Cramp::VERSION
+
+t_plan 4 "WebSocket monkey patch validity test for Cramp"
+
+CONFIG_RU=cramp/rainsocket.ru
+
+t_begin "setup and start" && {
+	rainbows_setup
+	rtmpfiles curl_err
+
+	# Like the rest of the EM/async stuff, it's not Rack::Lint compatible
+	rainbows -E deployment -D $CONFIG_RU -c $unicorn_config
+	rainbows_wait_start
+}
+
+t_begin "wait for server to say hello to us" && {
+	ok=$((curl --no-buffer -sS http://$listen/ || :) | \
+	     awk '/Hello from the Server/ { print "ok"; exit 0 }')
+
+	test x"$ok" = xok
+}
+
+t_begin "termination signal sent" && {
+	kill $rainbows_pid
+}
+
+t_begin "no errors in stderr" && check_stderr
+
+t_done