rainbows 0.91.1 → 0.92.0

data/lib/rainbows/max_body.rb

@@ -0,0 +1,82 @@
+# -*- encoding: binary -*-
+module Rainbows
+
+# middleware used to enforce client_max_body_size for TeeInput users,
+# there is no need to configure this middleware manually, it will
+# automatically be configured for you based on the client_max_body_size
+# setting
+class MaxBody < Struct.new(:app)
+
+  # this is meant to be included in Rainbows::TeeInput (and derived
+  # classes) to limit body sizes
+  module Limit
+    Util = Unicorn::Util
+
+    def initialize(socket, req, parser, buf)
+      self.len = parser.content_length
+
+      max = Rainbows.max_bytes # never nil, see MaxBody.setup
+      if len && len > max
+        socket.write(Const::ERROR_413_RESPONSE)
+        socket.close
+        raise IOError, "Content-Length too big: #{len} > #{max}", []
+      end
+
+      self.req = req
+      self.parser = parser
+      self.buf = buf
+      self.socket = socket
+      self.buf2 = ""
+      if buf.size > 0
+        parser.filter_body(buf2, buf) and finalize_input
+        buf2.size > max and raise IOError, "chunked request body too big", []
+      end
+      self.tmp = len && len < Const::MAX_BODY ? StringIO.new("") : Util.tmpio
+      if buf2.size > 0
+        tmp.write(buf2)
+        tmp.seek(0)
+        max -= buf2.size
+      end
+      @max_body = max
+    end
+
+    def tee(length, dst)
+      rv = super
+      if rv && ((@max_body -= rv.size) < 0)
+        # make HttpParser#keepalive? => false to force an immediate disconnect
+        # after we write
+        parser.reset
+        throw :rainbows_EFBIG
+      end
+      rv
+    end
+
+  end
+
+  # this is called after forking, so it won't ever affect the master
+  # if it's reconfigured
+  def self.setup
+    Rainbows.max_bytes or return
+    case G.server.use
+    when :Rev, :EventMachine, :NeverBlock
+      return
+    end
+
+    TeeInput.class_eval { include Limit }
+
+    # force ourselves to the outermost middleware layer
+    G.server.app = MaxBody.new(G.server.app)
+  end
+
+  # Rack response returned when there's an error
+  def err(env)
+    [ 413, [ %w(Content-Length 0), %w(Content-Type text/plain) ], [] ]
+  end
+
+  # our main Rack middleware endpoint
+  def call(env)
+    catch(:rainbows_EFBIG) { app.call(env) } || err(env)
+  end
+
+end # class
+end # module

data/lib/rainbows/rev/deferred_response.rb

@@ -10,9 +10,13 @@ module Rainbows
       G = Rainbows::G
       HH = Rack::Utils::HeaderHash
 
-      def self.defer!(client, response, out)
-        body = response.last
-        headers = HH.new(response[1])
+      def self.write(client, response, out)
+        status, headers, body = response
+
+        body.respond_to?(:to_path) or
+            return HttpResponse.write(client, response, out)
+
+        headers = HH.new(headers)
 
         # to_io is not part of the Rack spec, but make an exception
         # here since we can't get here without checking to_path first
@@ -39,16 +43,11 @@ module Rainbows
           headers.delete('Transfer-Encoding')
           headers['Content-Length'] ||= st.size.to_s
         else # char/block device, directory, whatever... nobody cares
-          return response
+          return HttpResponse.write(client, response, out)
         end
         client.defer_body(io, out)
-        [ response.first, headers.to_hash, [] ]
-      end
-
-      def self.write(client, response, out)
-        response.last.respond_to?(:to_path) and
-          response = defer!(client, response, out)
-        HttpResponse.write(client, response, out)
+        out.nil? or
+          client.write(HttpResponse.header_string(status, headers.to_hash, out))
       end
 
       def initialize(io, client, do_chunk, body)

data/lib/rainbows/revactor.rb

@@ -21,8 +21,6 @@ module Rainbows
   # concurrency features this model provides.
 
   module Revactor
-    require 'rainbows/revactor/tee_input'
-
     RD_ARGS = {}
 
     include Base
@@ -37,7 +35,7 @@ module Rainbows
         rd_args << RD_ARGS
         client.remote_addr
       else
-        LOCALHOST
+        Unicorn::HttpRequest::LOCALHOST
       end
       buf = client.read(*rd_args)
       hp = HttpParser.new
@@ -52,7 +50,7 @@ module Rainbows
         env[Const::CLIENT_IO] = client
         env[Const::RACK_INPUT] = 0 == hp.content_length ?
                  HttpRequest::NULL_IO :
-                 Rainbows::Revactor::TeeInput.new(client, env, hp, buf)
+                 TeeInput.new(PartialSocket.new(client), env, hp, buf)
         env[Const::REMOTE_ADDR] = remote_addr
         response = app.call(env.update(RACK_DEFAULTS))
 
@@ -137,5 +135,41 @@ module Rainbows
       end
     end
 
+    # Revactor Sockets do not implement readpartial, so we emulate just
+    # enough to avoid mucking with TeeInput internals.  Fortunately
+    # this code is not heavily used so we can usually avoid the overhead
+    # of adding a userspace buffer.
+    class PartialSocket < Struct.new(:socket, :rbuf)
+      def initialize(socket)
+        # IO::Buffer is used internally by Rev which Revactor is based on
+        # so we'll always have it available
+        super(socket, IO::Buffer.new)
+      end
+
+      # Revactor socket reads always return an unspecified amount,
+      # sometimes too much
+      def readpartial(length, dst = "")
+        return dst if length == 0
+        # always check and return from the userspace buffer first
+        rbuf.size > 0 and return dst.replace(rbuf.read(length))
+
+        # read off the socket since there was nothing in rbuf
+        tmp = socket.read
+
+        # we didn't read too much, good, just return it straight back
+        # to avoid needlessly wasting memory bandwidth
+        tmp.size <= length and return dst.replace(tmp)
+
+        # ugh, read returned too much, copy + reread to avoid slicing
+        rbuf << tmp[length, tmp.size]
+        dst.replace(tmp[0, length])
+      end
+
+      # just proxy any remaining methods TeeInput may use
+      def close
+        socket.close
+      end
+    end
+
   end
 end

data/lib/rainbows/tee_input.rb

@@ -0,0 +1,17 @@
+# -*- encoding: binary -*-
+module Rainbows
+
+  # acts like tee(1) on an input input to provide a input-like stream
+  # while providing rewindable semantics through a File/StringIO
+  # backing store.  On the first pass, the input is only read on demand
+  # so your Rack application can use input notification (upload progress
+  # and like).  This should fully conform to the Rack::InputWrapper
+  # specification on the public API.  This class is intended to be a
+  # strict interpretation of Rack::InputWrapper functionality and will
+  # not support any deviations from it.
+  class TeeInput < Unicorn::TeeInput
+
+    # empty class, this is to avoid unecessarily modifying Unicorn::TeeInput
+    # when MaxBody::Limit is included
+  end
+end

data/lib/rainbows.rb

@@ -22,15 +22,19 @@ module Rainbows
       false
     end
   end
+  # :stopdoc:
   G = State.new(true, 0, 0, 5)
   O = {}
+  # :startdoc:
 
   require 'rainbows/const'
   require 'rainbows/http_server'
   require 'rainbows/http_response'
   require 'rainbows/base'
+  require 'rainbows/tee_input'
   autoload :AppPool, 'rainbows/app_pool'
   autoload :DevFdResponse, 'rainbows/dev_fd_response'
+  autoload :MaxBody, 'rainbows/max_body'
 
   class << self
 
@@ -73,6 +77,20 @@ module Rainbows
       rv
     rescue Errno::EAGAIN, Errno::ECONNABORTED
     end
+
+    # returns a string representing the address of the given client +io+
+    # For local UNIX domain sockets, this will return a string referred
+    # to by the (non-frozen) Unicorn::HttpRequest::LOCALHOST constant.
+    def addr(io)
+      io.respond_to?(:peeraddr) ?
+                        io.peeraddr.last : Unicorn::HttpRequest::LOCALHOST
+    end
+
+    # the default max body size is 1 megabyte (1024 * 1024 bytes)
+    @@max_bytes = 1024 * 1024
+
+    def max_bytes; @@max_bytes; end
+    def max_bytes=(nr); @@max_bytes = nr; end
   end
 
   # configures \Rainbows! with a given concurrency model to +use+ and
@@ -83,6 +101,7 @@ module Rainbows
   #     use :Revactor # this may also be :ThreadSpawn or :ThreadPool
   #     worker_connections 400
   #     keepalive_timeout 0 # zero disables keepalives entirely
+  #     client_max_body_size 5*1024*1024 # 5 megabytes
   #   end
   #
   #   # the rest of the Unicorn configuration
@@ -94,16 +113,21 @@ module Rainbows
   # +worker_processes+ * +worker_connections+, so in the above example
   # we can serve 8 * 400 = 3200 clients concurrently.
   #
-  # The default is +keepalive_timeout+ is 2 seconds, which should be
+  # The default is +keepalive_timeout+ is 5 seconds, which should be
   # enough under most conditions for browsers to render the page and
   # start retrieving extra elements for.  Increasing this beyond 5
   # seconds is not recommended.  Zero disables keepalive entirely
   # (but pipelining fully-formed requests is still works).
+  #
+  # The default +client_max_body_size+ is 1 megabyte (1024 * 1024 bytes),
+  # setting this to +nil+ will disable body size checks and allow any
+  # size to be specified.
   def Rainbows!(&block)
     block_given? or raise ArgumentError, "Rainbows! requires a block"
     HttpServer.setup(block)
   end
 
+  # :stopdoc:
   # maps models to default worker counts, default worker count numbers are
   # pretty arbitrary and tuning them to your application and hardware is
   # highly recommended
@@ -116,7 +140,6 @@ module Rainbows
     :RevThreadSpawn => 50,
     :RevThreadPool => 50,
     :EventMachine => 50,
-    :EventMachineDefer => 50,
     :FiberSpawn => 50,
     :FiberPool => 50,
     :ActorSpawn => 50,
@@ -126,6 +149,7 @@ module Rainbows
     u = model.to_s.gsub(/([a-z0-9])([A-Z0-9])/) { "#{$1}_#{$2.downcase!}" }
     autoload model, "rainbows/#{u.downcase!}"
   end
+  # :startdoc:
   autoload :Fiber, 'rainbows/fiber' # core class
 
 end

data/local.mk.sample

@@ -6,39 +6,29 @@
 
 RSYNC = rsync
 DLEXT := so
-gems := rack-1.1.0
-# gems += unicorn-0.96.0 # installed via setup.rb
-gems += rev-0.3.2
-gems += iobuffer-0.1.3
-gems += eventmachine-0.12.10
-gems += async_sinatra-0.1.5 sinatra-0.9.4
-gems += espace-neverblock-0.1.6.1
-
-# Cramp isn't enabled by default since it depends on several prerelease gems
-ifdef CRAMP
-  gems += cramp-0.7
-  gems += activesupport-3.0.pre
-  gems += activemodel-3.0.pre
-  gems += arel-0.2.pre
-  gems += usher-0.6.2
-  gems += fuzzyhash-0.0.11
-  gems += mysqlplus-0.1.1
-endif
 
 # Avoid loading rubygems to speed up tests because gmake is
 # fork+exec heavy with Ruby.
 prefix = $(HOME)
+
 ifeq ($(r19),)
   RUBY := $(prefix)/bin/ruby
-  gem_paths := $(addprefix $(prefix)/lib/ruby/gems/1.8/gems/,$(gems))
 else
   prefix := $(prefix)/ruby-1.9
   export PATH := $(prefix)/bin:$(PATH)
   RUBY := $(prefix)/bin/ruby --disable-gems
-  gems += case-0.5 revactor-0.1.5
-  gem_paths := $(addprefix $(prefix)/lib/ruby/gems/1.9.1/gems/,$(gems))
 endif
 
+ifndef NO_ISOLATE
+  x := $(shell test -d t/ && \
+         PATH=$(PATH) NO_ISOLATE=T $(MAKE) -s isolate RUBY:="$(RUBY)")
+endif
+
+RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
+
+updir := $(shell git rev-parse --show-cdup)
+gem_paths := $(wildcard $(updir)tmp/gems/$(RUBY_VERSION)/gems/*-*)
+
 ifdef gem_paths
   sp :=
   sp +=
@@ -55,9 +45,9 @@ TRACER = /usr/bin/time -v -o $(t_pfx).time
 
 full-test: test-18 test-19
 test-18:
-	$(MAKE) test 2>&1 | sed -u -e 's!^!1.8 !'
+	$(MAKE) test 2>&1 | sed -e 's!^!1.8 !'
 test-19:
-	$(MAKE) test r19=t 2>&1 | sed -u -e 's!^!1.9 !'
+	$(MAKE) test r19=T 2>&1 | sed -e 's!^!1.9 !'
 
 latest: NEWS
 	@awk 'BEGIN{RS="=== ";ORS=""}NR==2{sub(/\n$$/,"");print RS""$$0 }' < $<

data/rainbows.gemspec

@@ -43,13 +43,13 @@ Gem::Specification.new do |s|
   # we need Unicorn for the HTTP parser and process management
   # The HTTP parser in Unicorn <= 0.97.0 was vulnerable to a remote DoS
   # when exposed directly to untrusted clients.
-  s.add_dependency(%q<unicorn>, ["~> 0.97.1"])
+  s.add_dependency(%q<unicorn>, [">= 0.97.1", "< 1.0.0"])
 
   # Unicorn already depends on Rack
   # s.add_dependency(%q<rack>)
 
   # optional runtime dependencies depending on configuration
-  # see local.mk.sample for the exact versions we've tested with
+  # see config/isolate.rb for the exact versions we've tested with
   #
   # Revactor >= 0.1.5 includes UNIX domain socket support
   # s.add_dependency(%q<revactor>, [">= 0.1.5"])

data/t/app_deferred.ru

@@ -0,0 +1,23 @@
+#\-E none
+# can't use non-compatible middleware that doesn't pass "deferered?" calls
+#
+# used for testing deferred actions for Merb and possibly other frameworks
+# ref: http://brainspl.at/articles/2008/04/18/deferred-requests-with-merb-ebb-and-thin
+
+class DeferredApp < Struct.new(:app)
+  def deferred?(env)
+    env["PATH_INFO"] == "/deferred"
+  end
+
+  def call(env)
+    env["rack.multithread"] or raise RuntimeError, "rack.multithread not true"
+    body = "#{Thread.current.inspect}\n"
+    headers = {
+      "Content-Type" => "text/plain",
+      "Content-Length" => body.size.to_s,
+    }
+    [ 200, headers, [ body ] ]
+  end
+end
+
+run DeferredApp.new

data/t/async_examples/async_app.ru

@@ -89,16 +89,16 @@ class AsyncApp
     # Get the headers out there asap, let the client know we're alive...
     EventMachine::next_tick { env['async.callback'].call [200, {'Content-Type' => 'text/plain'}, body] }
 
-    # Semi-emulate a long db request, instead of a timer, in reality we'd be
-    # waiting for the response data. Whilst this happens, other connections
+    # Semi-emulate a long db request, instead of a timer, in reality we'd be
+    # waiting for the response data. Whilst this happens, other connections
     # can be serviced.
     # This could be any callback based thing though, a deferrable waiting on
-    # IO data, a db request, an http request, an smtp send, whatever.
+    # IO data, a db request, an http request, an smtp send, whatever.
     EventMachine::add_timer(1) {
       body.call ["Woah, async!\n"]
 
       EventMachine::next_tick {
-        # This could actually happen any time, you could spawn off to new
+        # This could actually happen any time, you could spawn off to new
         # threads, pause as a good looking lady walks by, whatever.
         # Just shows off how we can defer chunks of data in the body, you can
         # even call this many times.
@@ -116,11 +116,11 @@ end
 
 # The additions to env for async.connection and async.callback absolutely
 # destroy the speed of the request if Lint is doing it's checks on env.
-# It is also important to note that an async response will not pass through
-# any further middleware, as the async response notification has been passed
-# right up to the webserver, and the callback goes directly there too.
+# It is also important to note that an async response will not pass through
+# any further middleware, as the async response notification has been passed
+# right up to the webserver, and the callback goes directly there too.
 # Middleware could possibly catch :async, and also provide a different
-# async.connection and async.callback.
+# async.connection and async.callback.
 
 # use Rack::Lint
 run AsyncApp.new

data/t/rack-fiber_pool/app.ru

@@ -0,0 +1,5 @@
+require 'rack/fiber_pool'
+use Rack::FiberPool
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+run lambda { |env| [ 200, {}, [ "#{Fiber.current}\n" ] ] }

data/t/{t0100-rack-input-hammer.sh → t0100-rack-input-hammer-chunked.sh}

@@ -7,7 +7,7 @@ test -r random_blob || die "random_blob required, run with 'make $0'"
 # So we try to use things like curl and sha1sum that are implemented
 # without the Ruby interpreter to validate our own Ruby internals.
 
-t_plan 7 "concurrent rack.input hammer stress test"
+t_plan 7 "concurrent rack.input hammer stress test (chunked)"
 
 t_begin "setup and startup" && {
 	rtmpfiles curl_out curl_err

data/t/t0100-rack-input-hammer-content-length.sh

@@ -0,0 +1,50 @@
+nr_client=${nr_client-4}
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+
+# basically we don't trust our own implementation of content-md5-put
+# nor our Ruby 1.9 knowledge nor proper use of encodings in Ruby.
+# So we try to use things like curl and sha1sum that are implemented
+# without the Ruby interpreter to validate our own Ruby internals.
+
+t_plan 7 "concurrent rack.input hammer stress test (content-length)"
+
+t_begin "setup and startup" && {
+	rtmpfiles curl_out curl_err
+	rainbows_setup $model
+	rainbows -D sha1.ru -c $unicorn_config
+	rainbows_wait_start
+}
+
+t_begin "send $nr_client concurrent requests" && {
+	start=$(date +%s)
+	for i in $(awk "BEGIN{for(i=0;i<$nr_client;++i) print i}" </dev/null)
+	do
+		(
+			curl -sSf -T random_blob http://$listen/$i \
+			  >> $curl_out 2>> $curl_err
+		) &
+	done
+	wait
+	t_info elapsed=$(( $(date +%s) - $start ))
+}
+
+t_begin "kill server" && kill $rainbows_pid
+
+t_begin "got $nr_client responses" && {
+	test $nr_client -eq $(wc -l < $curl_out)
+}
+
+t_begin "all responses identical" && {
+	test 1 -eq $(sort < $curl_out | uniq | wc -l)
+}
+
+t_begin "sha1 matches on-disk sha1" && {
+	blob_sha1=$(rsha1 < random_blob)
+	t_info blob_sha1=$blob_sha1
+	test x"$blob_sha1" = x"$(sort < $curl_out | uniq)"
+}
+
+t_begin "no errors in stderr log" && check_stderr
+
+t_done

data/t/t0103-rack-input-limit.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+
+t_plan 6 "rack.input client_max_body_size default"
+
+t_begin "setup and startup" && {
+	rtmpfiles curl_out curl_err cmbs_config
+	rainbows_setup $model
+	grep -v client_max_body_size < $unicorn_config > $cmbs_config
+	rainbows -D sha1-random-size.ru -c $cmbs_config
+	rainbows_wait_start
+}
+
+t_begin "regular request" && {
+	rm -f $ok
+	curl -vsSf -T random_blob -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err || > $ok
+	dbgcat curl_err
+	dbgcat curl_out
+	test -e $ok
+}
+
+t_begin "chunked request" && {
+	rm -f $ok
+	curl -vsSf -T- < random_blob -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err || > $ok
+	dbgcat curl_err
+	dbgcat curl_out
+	test -e $ok
+}
+
+t_begin "default size sha1 chunked" && {
+	blob_sha1=3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3
+	rm -f $ok
+	> $r_err
+	dd if=/dev/zero bs=1048576 count=1 | \
+	  curl -vsSf -T- -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err
+	test "$(cat $curl_out)" = $blob_sha1
+	dbgcat curl_err
+	dbgcat curl_out
+}
+
+t_begin "default size sha1 content-length" && {
+	blob_sha1=3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3
+	rm -f $ok
+	dd if=/dev/zero bs=1048576 count=1 of=$tmp
+	curl -vsSf -T $tmp -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err
+	test "$(cat $curl_out)" = $blob_sha1
+	dbgcat curl_err
+	dbgcat curl_out
+}
+
+t_begin "shutdown" && {
+	kill $rainbows_pid
+}
+
+t_done

data/t/t0104-rack-input-limit-tiny.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+
+t_plan 6 "rack.input client_max_body_size tiny"
+
+t_begin "setup and startup" && {
+	rtmpfiles curl_out curl_err cmbs_config
+	rainbows_setup $model
+	sed -e 's/client_max_body_size.*/client_max_body_size 256/' \
+	  < $unicorn_config > $cmbs_config
+	rainbows -D sha1-random-size.ru -c $cmbs_config
+	rainbows_wait_start
+}
+
+t_begin "stops a regular request" && {
+	rm -f $ok
+	dd if=/dev/zero bs=257 count=1 of=$tmp
+	curl -vsSf -T $tmp -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err || > $ok
+	dbgcat curl_err
+	dbgcat curl_out
+	test -e $ok
+}
+
+t_begin "stops a large chunked request" && {
+	rm -f $ok
+	dd if=/dev/zero bs=257 count=1 | \
+	  curl -vsSf -T- -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err || > $ok
+	dbgcat curl_err
+	dbgcat curl_out
+	test -e $ok
+}
+
+t_begin "small size sha1 chunked ok" && {
+	blob_sha1=b376885ac8452b6cbf9ced81b1080bfd570d9b91
+	rm -f $ok
+	dd if=/dev/zero bs=256 count=1 | \
+	  curl -vsSf -T- -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err
+	dbgcat curl_err
+	dbgcat curl_out
+	test "$(cat $curl_out)" = $blob_sha1
+}
+
+t_begin "small size sha1 content-length ok" && {
+	blob_sha1=b376885ac8452b6cbf9ced81b1080bfd570d9b91
+	rm -f $ok
+	dd if=/dev/zero bs=256 count=1 of=$tmp
+	curl -vsSf -T $tmp -H Expect: \
+	  http://$listen/ > $curl_out 2> $curl_err
+	dbgcat curl_err
+	dbgcat curl_out
+	test "$(cat $curl_out)" = $blob_sha1
+}
+
+t_begin "shutdown" && {
+	kill $rainbows_pid
+}
+
+t_done