rainbows 0.3.0 → 0.4.0

data/lib/rainbows/http_server.rb

@@ -33,7 +33,10 @@ module Rainbows
       extend(mod)
       Const::RACK_DEFAULTS['rainbows.model'] = @use = model
       Const::RACK_DEFAULTS['rack.multithread'] = !!(/Thread/ =~ model.to_s)
-      Const::RACK_DEFAULTS['rainbows.autochunk'] = (model.to_s == "Rev")
+      case model
+      when :Rev, :EventMachine
+        Const::RACK_DEFAULTS['rainbows.autochunk'] = true
+      end
     end
 
     def worker_connections(*args)

data/lib/rainbows/rev.rb

@@ -1,9 +1,7 @@
 # -*- encoding: binary -*-
 require 'rev'
-
-# workaround revactor 0.1.4 still using the old Rev::Buffer
-# ref: http://rubyforge.org/pipermail/revactor-talk/2009-October/000034.html
-defined?(Rev::Buffer) or Rev::Buffer = IO::Buffer
+Rev::VERSION >= '0.3.0' or abort 'rev >= 0.3.0 is required'
+require 'rainbows/ev_core'
 
 module Rainbows
 
@@ -30,46 +28,22 @@ module Rainbows
     include Base
 
     class Client < ::Rev::IO
-      include Unicorn
-      include Rainbows::Const
+      include Rainbows::EvCore
       G = Rainbows::G
 
-      # queued, optional response bodies, it should only be unpollable "fast"
-      # devices where read(2) is uninterruptable.  Unfortunately, NFS and ilk
-      # are also part of this.  We'll also stick DeferredResponse bodies in
-      # here to prevent connections from being closed on us.
-      attr_reader :deferred_bodies
-
       def initialize(io)
         G.cur += 1
         super(io)
-        @remote_addr = ::TCPSocket === io ? io.peeraddr.last : LOCALHOST
-        @env = {}
-        @hp = HttpParser.new
-        @state = :headers # [ :body [ :trailers ] ] :app_call :close
-        @buf = ""
-        @deferred_bodies = [] # for (fast) regular files only
-      end
-
-      # graceful exit, like SIGQUIT
-      def quit
-        @deferred_bodies.clear
-        @state = :close
+        post_init
       end
 
-      def handle_error(e)
-        quit
-        msg = case e
-        when EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::EINVAL,Errno::EBADF
-          ERROR_500_RESPONSE
-        when HttpParserError # try to tell the client they're bad
-          ERROR_400_RESPONSE
-        else
-          G.logger.error "Read error: #{e.inspect}"
-          G.logger.error e.backtrace.join("\n")
-          ERROR_500_RESPONSE
-        end
-        write(msg)
+      # queued, optional response bodies, it should only be unpollable "fast"
+      # devices where read(2) is uninterruptable.  Unfortunately, NFS and ilk
+      # are also part of this.  We'll also stick DeferredResponse bodies in
+      # here to prevent connections from being closed on us.
+      def defer_body(io)
+        @deferred_bodies << io
+        on_write_complete unless @hp.headers? # triggers a write
       end
 
       def app_call
@@ -89,7 +63,7 @@ module Rainbows
             # keepalive requests are always body-less, so @input is unchanged
             @hp.headers(@env, @buf) and next
           else
-            @state = :close
+            quit
           end
           return
         end while true
@@ -104,6 +78,7 @@ module Rainbows
             rescue EOFError # expected at file EOF
               @deferred_bodies.shift
               body.close
+              close if :close == @state && @deferred_bodies.empty?
             end
           rescue Object => e
             handle_error(e)
@@ -116,53 +91,6 @@ module Rainbows
       def on_close
         G.cur -= 1
       end
-
-      def tmpio
-        io = Util.tmpio
-        def io.size
-          # already sync=true at creation, so no need to flush before stat
-          stat.size
-        end
-        io
-      end
-
-      # TeeInput doesn't map too well to this right now...
-      def on_read(data)
-        case @state
-        when :headers
-          @hp.headers(@env, @buf << data) or return
-          @state = :body
-          len = @hp.content_length
-          if len == 0
-            @input = HttpRequest::NULL_IO
-            app_call # common case
-          else # nil or len > 0
-            # since we don't do streaming input, we have no choice but
-            # to take over 100-continue handling from the Rack application
-            if @env[HTTP_EXPECT] =~ /\A100-continue\z/i
-              write(EXPECT_100_RESPONSE)
-              @env.delete(HTTP_EXPECT)
-            end
-            @input = len && len <= MAX_BODY ? StringIO.new("") : tmpio
-            @hp.filter_body(@buf2 = @buf.dup, @buf)
-            @input << @buf2
-            on_read("")
-          end
-        when :body
-          if @hp.body_eof?
-            @state = :trailers
-            on_read(data)
-          elsif data.size > 0
-            @hp.filter_body(@buf2, @buf << data)
-            @input << @buf2
-            on_read("")
-          end
-        when :trailers
-          @hp.trailers(@env, @buf << data) and app_call
-        end
-        rescue Object => e
-          handle_error(e)
-      end
     end
 
     class Server < ::Rev::IO
@@ -172,7 +100,7 @@ module Rainbows
         return if G.cur >= G.max
         begin
           Client.new(@_io.accept_nonblock).attach(::Rev::Loop.default)
-        rescue Errno::EAGAIN, Errno::ECONNBORTED
+        rescue Errno::EAGAIN, Errno::ECONNABORTED
         end
       end
 
@@ -191,7 +119,7 @@ module Rainbows
         # here since we can't get here without checking to_path first
         io = body.to_io if body.respond_to?(:to_io)
         io ||= ::IO.new($1.to_i) if body.to_path =~ %r{\A/dev/fd/(\d+)\z}
-        io ||= File.open(File.expand_path(body.to_path), 'rb')
+        io ||= File.open(body.to_path, 'rb')
         st = io.stat
 
         if st.socket? || st.pipe?
@@ -199,7 +127,11 @@ module Rainbows
           do_chunk = false if headers.delete('X-Rainbows-Autochunk') == 'no'
           # too tricky to support keepalive/pipelining when a response can
           # take an indeterminate amount of time here.
-          out[0] = CONN_CLOSE
+          if out.nil?
+            do_chunk = false
+          else
+            out[0] = CONN_CLOSE
+          end
 
           io = new(io, client, do_chunk, body).attach(::Rev::Loop.default)
         elsif st.file?
@@ -208,7 +140,7 @@ module Rainbows
         else # char/block device, directory, whatever... nobody cares
           return response
         end
-        client.deferred_bodies << io
+        client.defer_body(io)
         [ response.first, headers.to_hash, [] ]
       end
 

data/lib/rainbows/revactor.rb

@@ -1,9 +1,6 @@
 # -*- encoding: binary -*-
 require 'revactor'
-
-# workaround revactor 0.1.4 still using the old Rev::Buffer
-# ref: http://rubyforge.org/pipermail/revactor-talk/2009-October/000034.html
-defined?(Rev::Buffer) or Rev::Buffer = IO::Buffer
+Revactor::VERSION >= '0.1.5' or abort 'revactor 0.1.5 is required'
 
 module Rainbows
 
@@ -137,16 +134,13 @@ module Rainbows
 
     def revactorize_listeners!
       LISTENERS.map! do |s|
-        if TCPServer === s
+        case s
+        when TCPServer
           ::Revactor::TCP.listen(s, nil)
-        elsif defined?(::Revactor::UNIX) && UNIXServer === s
+        when UNIXServer
           ::Revactor::UNIX.listen(s)
-        else
-          logger.error "your version of Revactor can't handle #{s.inspect}"
-          nil
         end
       end
-      LISTENERS.compact!
     end
 
   end

data/local.mk.sample

@@ -5,18 +5,24 @@
 # This is depends on a bunch of GNU-isms from bash, sed, touch.
 
 DLEXT := so
-gems := rev-0.3.1 rack-1.0.0 iobuffer-0.1.1
+gems := rack-1.0.1
+# gems += unicorn-0.93.3 # installed via setup.rb
+gems += rev-0.3.1 iobuffer-0.1.1
+gems += eventmachine-0.12.10
+gems += async_sinatra-0.1.5 sinatra-0.9.4
 
 # Avoid loading rubygems to speed up tests because gmake is
 # fork+exec heavy with Ruby.
+prefix = $(HOME)
 ifeq ($(r19),)
-  ruby := $(HOME)/bin/ruby
-  gem_paths := $(addprefix $(HOME)/lib/ruby/gems/1.8/gems/,$(gems))
+  RUBY := $(prefix)/bin/ruby
+  gem_paths := $(addprefix $(prefix)/lib/ruby/gems/1.8/gems/,$(gems))
 else
-  export PATH := $(HOME)/ruby-1.9/bin:$(PATH)
-  ruby := $(HOME)/ruby-1.9/bin/ruby --disable-gems
-  gems := $(gems) case-0.5 revactor-0.1.4
-  gem_paths := $(addprefix $(HOME)/ruby-1.9/lib/ruby/gems/1.9.1/gems/,$(gems))
+  prefix := $(prefix)/ruby-1.9
+  export PATH := $(prefix)/bin:$(PATH)
+  RUBY := $(prefix)/bin/ruby --disable-gems
+  gems += case-0.5 revactor-0.1.5
+  gem_paths := $(addprefix $(prefix)/lib/ruby/gems/1.9.1/gems/,$(gems))
 endif
 
 ifdef gem_paths

data/rainbows.gemspec

@@ -40,8 +40,23 @@ Gem::Specification.new do |s|
 
   s.test_files = test_files
 
-  s.add_dependency(%q<rack>)
-  s.add_dependency(%q<unicorn>, ["~> 0.93.1"])
+  # we need Unicorn for the HTTP parser and process management
+  s.add_dependency(%q<unicorn>, ["~> 0.93.4"])
+
+  # Unicorn already depends on Rack
+  # s.add_dependency(%q<rack>)
+
+  # optional runtime dependencies depending on configuration
+  # see local.mk.sample for the exact versions we've tested with
+  #
+  # Revactor >= 0.1.5 includes UNIX domain socket support
+  # s.add_dependency(%q<revactor>, [">= 0.1.5"])
+  #
+  # Revactor depends on Rev, too, 0.3.0 got the ability to attach IOs
+  # s.add_dependency(%q<rev>, [">= 0.3.0"])
+  #
+  # We use the new EM::attach/watch API in 0.12.10
+  # s.add_dependency(%q<eventmachine>, ["~> 0.12.10"])
 
   # s.licenses = %w(GPLv2 Ruby) # accessor not compatible with older Rubygems
 end

data/t/.gitignore

@@ -1,3 +1,4 @@
 /test-results-*
 /test-bin-*
 /random_blob
+/.dep+*

data/t/GNUmakefile

@@ -2,11 +2,13 @@
 
 all::
 
-ruby = ruby
+pid := $(shell echo $$PPID)
+
+RUBY = $(ruby)
 rainbows_lib := $(shell cd ../lib && pwd)
 -include ../local.mk
 ifeq ($(RUBY_VERSION),)
-  RUBY_VERSION := $(shell $(ruby) -e 'puts RUBY_VERSION')
+  RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
 endif
 
 ifeq ($(RUBYLIB),)
@@ -16,60 +18,81 @@ else
 endif
 export RUBYLIB RUBY_VERSION
 
+models := ThreadPool ThreadSpawn Rev EventMachine
+ifeq ($(RUBY_VERSION),1.9.1) # 1.9.2-preview1 was broken
+  models += Revactor
+endif
+all_models := $(models) Base
+
 T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
 
+MODEL_T := $(foreach m,$(all_models),$(addprefix $(m).,$(T)))
+$(T): MODELS = $(models)
+
+# some tests can be run with all models
+t0000-simple-http.sh: MODELS = $(all_models)
+t0001-unix-http.sh: MODELS = $(all_models)
+t0002-graceful.sh: MODELS = $(all_models)
+t0002-parser-error.sh: MODELS = $(all_models)
+t0003-reopen-logs.sh: MODELS = $(all_models)
+
+# this test is not compatible with non-Thread models yet
+t9000-rack-app-pool.sh: MODELS = ThreadPool ThreadSpawn
+
+# recursively run per-model tests
+# haven't figured out a good way to make make non-recursive here, yet...
+$(T):
+	$(MAKE) $(foreach m,$(MODELS),$(addprefix $(m).,$@))
+
+$(all_models):
+	$(MAKE) $(filter $@.%,$(MODEL_T))
+
 all:: $(T)
 
 # can't rely on "set -o pipefail" since we don't require bash or ksh93 :<
 t_pfx = trash/$@-$(RUBY_VERSION)
-t_code = $(t_pfx).code
-t_log = $(t_pfx).log
+TEST_OPTS =
 # TRACER = strace -f -o $(t_pfx).strace -s 100000
 # TRACER = /usr/bin/time -o $(t_pfx).time
-t_run = $(TRACER) $(SHELL) $(SH_TEST_OPTS) $@
-
-# prefix stdout messages with ':', and stderr messages with '!'
-t_wrap = ( ( ( echo 42 > $(t_code); \
-  $(t_run); \
-  echo $$? > $(t_code) ) \
-  | sed 's/^/$(pfx):/' 1>&3 ) 2>&1 \
-  | sed 's/^/$(pfx)!/' 1>&2 ) 3>&1
-
-ifndef V
-  quiet_pre = @echo '* $@';
-  quiet_post = > $(t_log) 2>&1; exit $$(cat $(t_code))
-  pfx =
-else
+
+ifdef V
   ifeq ($(V),2)
-    SH_TEST_OPTS += -x
+    TEST_OPTS += --trace
+  else
+    TEST_OPTS += --verbose
   endif
-  quiet_pre = @echo '* $@';
-  quiet_post = 2>&1 | ./bin/utee $(t_log); exit $$(cat $(t_code))
-  pfx = $@
 endif
 
-run_test = $(quiet_pre) ( $(t_wrap) ) $(quiet_post)
-
-test-bin-$(RUBY_VERSION)/rainbows: ruby_bin = $(shell which $(ruby))
+test-bin-$(RUBY_VERSION)/rainbows: ruby_bin = $(shell which $(RUBY))
 test-bin-$(RUBY_VERSION)/rainbows: ../bin/rainbows
 	mkdir -p $(@D)
-	install -m 755 $^ $@+
-	$(ruby) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@+
-	cmp $@+ $@ 2>/dev/null || mv $@+ $@
-	$(RM) $@+
+	install -m 755 $^ $@.$(pid)
+	$(RUBY) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@.$(pid)
+	mv $@.$(pid) $@
 
-req_random_blob := $(wildcard t?1??-*.sh)
 random_blob:
-	dd if=/dev/urandom bs=1M count=10 of=$@+
-	mv $@+ $@
-
-$(req_random_blob): random_blob
-
-$(T): trash/.gitignore
-$(T): export ruby := $(ruby)
-$(T): export PATH := $(CURDIR)/test-bin-$(RUBY_VERSION):$(PATH)
-$(T): test-bin-$(RUBY_VERSION)/rainbows
-	$(run_test)
+	dd if=/dev/urandom bs=1M count=30 of=$@.$(pid)
+	mv $@.$(pid) $@
+
+$(T): random_blob
+
+dependencies := socat curl
+deps := $(addprefix .dep+,$(dependencies))
+$(deps): dep_bin = $(lastword $(subst +, ,$@))
+$(deps):
+	@which $(dep_bin) > $@.$(pid) 2>/dev/null || :
+	@test -s $@.$(pid) || \
+	  { echo >&2 "E `$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
+	@mv $@.$(pid) $@
+dep: $(deps)
+
+$(MODEL_T): export model = $(firstword $(subst ., ,$@))
+$(MODEL_T): script = $(subst $(model).,,$@)
+$(MODEL_T): trash/.gitignore
+$(MODEL_T): export RUBY := $(RUBY)
+$(MODEL_T): export PATH := $(CURDIR)/test-bin-$(RUBY_VERSION):$(PATH)
+$(MODEL_T): test-bin-$(RUBY_VERSION)/rainbows dep
+	$(TRACER) $(SHELL) $(SH_TEST_OPTS) $(script) $(TEST_OPTS)
 
 trash/.gitignore:
 	mkdir -p $(@D)

data/t/README

@@ -28,9 +28,13 @@ To run the entire test suite with 8 tests running at once:
 
   make -j8
 
-To run one individual test:
+To run one individual test for all concurrency models:
 
-  make t0000-basic.sh
+  make t0000-simple-http.sh
+
+To run one individual test for one concurrency model:
+
+  make Revactor.t0000-simple-http.sh
 
 You may also increase verbosity by setting the "V" variable for
 GNU make.  To disable trapping of stdout/stderr:

data/t/async_sinatra.ru

@@ -0,0 +1,13 @@
+# See http://github.com/raggi/async_sinatra
+# gem install async_sinatra -v0.1.5
+require 'sinatra/async'
+
+class AsyncTest < Sinatra::Base
+  register Sinatra::Async
+
+  aget '/:n' do |n|
+    EM.add_timer(n.to_i) { body { "delayed for #{n} seconds\n" } }
+  end
+end
+
+run AsyncTest.new