rainbows 0.91.1 → 0.92.0

data/Documentation/comparison.haml

@@ -274,7 +274,7 @@
     %td.devfd Yes
     %td.app_pool no-op
     %td.lock no-op
-    %td.async async_sinatra
+    %td.async async_sinatra, Cramp, rack-fiber_pool
     %td.ws no
   %tr.comp_row
     %td.mod RevThreadSpawn

data/GIT-VERSION-FILE

@@ -1 +1 @@
-GIT_VERSION = 0.91.1
+GIT_VERSION = 0.92.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.91.1.GIT
+DEF_VER=v0.92.0.GIT
 
 LF='
 '

data/GNUmakefile

@@ -3,6 +3,7 @@ all::
 RUBY = ruby
 RAKE = rake
 GIT_URL = git://git.bogomips.org/rainbows.git
+ISOLATE_CONFIG = config/isolate.rb
 
 GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
 	@./GIT-VERSION-GEN
@@ -15,8 +16,15 @@ ifeq ($(RUBY_VERSION),)
   RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
 endif
 
+# rake takes forever to start
+isolate: tmp/gems/$(RUBY_VERSION)/.isolate
+tmp/gems/$(RUBY_VERSION)/.isolate: $(ISOLATE_CONFIG)
+	ISOLATE_CONFIG=$(ISOLATE_CONFIG) $(RAKE) isolate
+	> $@
+
 base_bins := rainbows
 bins := $(addprefix bin/, $(base_bins))
+man1_rdoc := $(addsuffix _1, $(base_bins))
 man1_bins := $(addsuffix .1, $(base_bins))
 man1_paths := $(addprefix man/man1/, $(man1_bins))
 
@@ -58,7 +66,7 @@ NEWS: GIT-VERSION-FILE
 	$(RAKE) -s news_rdoc > $@+
 	mv $@+ $@
 
-SINCE = 0.90.0
+SINCE = 0.91.1
 ChangeLog: LOG_VERSION = \
   $(shell git rev-parse -q "$(GIT_VERSION)" >/dev/null 2>&1 && \
           echo $(GIT_VERSION) || git describe)
@@ -74,18 +82,21 @@ cgit_atom := http://git.bogomips.org/cgit/rainbows.git/atom/?h=master
 atom = <link rel="alternate" title="Atom feed" href="$(1)" \
              type="application/atom+xml"/>
 
-# using rdoc 2.4.1+
+# using rdoc 2.5.x+
 doc: .document NEWS ChangeLog
-	for i in $(man1_bins); do > $$i; done
+	for i in $(man1_rdoc); do echo > $$i; done
 	find bin lib -type f -name '*.rbc' -exec rm -f '{}' ';'
-	rdoc -Na -t "$(shell sed -ne '1s/^= //p' README)"
+	rdoc -a -t "$(shell sed -ne '1s/^= //p' README)"
 	install -m644 COPYING doc/COPYING
 	install -m644 $(shell grep '^[A-Z]' .document)  doc/
 	$(MAKE) -C Documentation install-html install-man
 	install -m644 $(man1_paths) doc/
 	cd doc && for i in $(base_bins); do \
+	  $(RM) 1.html $${i}.1.html; \
 	  sed -e '/"documentation">/r man1/'$$i'.1.html' \
-		< $${i}_1.html > tmp && mv tmp $${i}_1.html; done
+		< $${i}_1.html > tmp && mv tmp $${i}_1.html; \
+	  ln $${i}_1.html $${i}.1.html; \
+	  done
 	$(RUBY) -i -p -e \
 	  '$$_.gsub!("</title>",%q{\&$(call atom,$(cgit_atom))})' \
 	  doc/ChangeLog.html
@@ -99,7 +110,7 @@ doc: .document NEWS ChangeLog
 	  '$$_.gsub!(/INCLUDE/){File.read("Documentation/comparison.html")}' \
 	  doc/Summary.html
 	cat Documentation/comparison.css >> doc/rdoc.css
-	$(RM) $(man1_bins)
+	$(RM) $(man1_rdoc)
 
 ifneq ($(VERSION),)
 rfproject := rainbows
@@ -156,7 +167,7 @@ release: verify package $(release_notes) $(release_changes)
 	# make tgz release on RubyForge
 	rubyforge add_release -f -n $(release_notes) -a $(release_changes) \
 	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
-	# push gem to Gemcutter
+	# push gem to RubyGems.org
 	gem push $(pkggem)
 	# in case of gem downloads from RubyForge releases page
 	-rubyforge add_file \

data/NEWS

@@ -1,3 +1,32 @@
+=== 0.92.0 / 2010-05-04 21:58 UTC
+
+  Mostly internal cleanups and small improvements.
+
+  The only backwards incompatible change was the addition of the
+  "client_max_body_size" parameter to limit upload sizes to
+  prevent DoS.  This defaults to one megabyte (same as nginx), so
+  any apps relying on the limit-less behavior of previous will
+  have to configure this in the Unicorn/Rainbows! config file:
+
+        Rainbows! do
+  	# nil for unlimited, or any number in bytes
+  	client_max_body_size nil
+        end
+
+  The ThreadSpawn and ThreadPool models are now optimized for serving
+  large static files under Ruby 1.9 using IO.copy_stream[1].
+
+  The EventMachine model has always had optimized static file
+  serving (using EM::Connection#stream_file_data[2]).
+
+  The EventMachine model (finally) gets conditionally deferred app
+  dispatch in a separate thread, as described by Ezra Zygmuntowicz
+  for Merb, Ebb and Thin[3].
+
+  [1] - http://euruko2008.csrug.cz/system/assets/documents/0000/0007/tanaka-IOcopy_stream-euruko2008.pdf
+  [2] - http://eventmachine.rubyforge.org/EventMachine/Connection.html#M000312
+  [3] - http://brainspl.at/articles/2008/04/18/deferred-requests-with-merb-ebb-and-thin
+
 === 0.91.1 / 2010-04-19 21:13 UTC
 
   This release fixes a denial-of-service vector for deployments

data/README

@@ -1,6 +1,6 @@
-= Rainbows! Unicorn for sleepy apps and slow clients
+= Rainbows! - Unicorn for sleepy apps and slow clients
 
-Rainbows! is an HTTP server for sleepy Rack applications.  It is based on
+\Rainbows! is an HTTP server for sleepy Rack applications.  It is based on
 Unicorn, but designed to handle applications that expect long
 request/response times and/or slow clients.  For Rack applications not
 heavily bound by slow external network dependencies, consider Unicorn
@@ -92,7 +92,7 @@ and run setup.rb after unpacking it:
 
 http://rubyforge.org/frs/?group_id=8977
 
-You may also install it via RubyGems on Gemcutter:
+You may also install it via RubyGems on RubyGems.org:
 
   gem install rainbows
 
@@ -123,7 +123,8 @@ config file:
       worker_connections 100
     end
 
-See the {Rainbows! configuration documentation}[link:Rainbows.html#M000001]
+See the {Rainbows! configuration}[link:Rainbows.html#method-i-Rainbows!]
+{documentation}[link:Rainbows.html#method-i-Rainbows!]
 for more details.
 
 == Development

data/Rakefile

@@ -183,3 +183,10 @@ task :fm_update do
     p http.post(uri.path, req, {'Content-Type'=>'application/json'})
   end
 end
+
+desc 'isolate gems for development'
+task :isolate do
+  require 'isolate'
+  Isolate.gems "tmp/gems/#{RUBY_VERSION}",
+               :file => ENV['ISOLATE_CONFIG']
+end

data/SIGNALS

@@ -14,7 +14,9 @@ between \Rainbows!, Unicorn and nginx.
 * INT/TERM - quick shutdown, kills all workers immediately
 
 * QUIT - graceful shutdown, waits for workers to finish their
-  current request before finishing.
+  current request before finishing.  This currently does not
+  wait for requests deferred to a separate thread when using
+  EventMachine (when app.deferred?(env) => true)
 
 * USR1 - reopen all logs owned by the master and all workers
   See Unicorn::Util.reopen_logs for what is considered a log.
@@ -43,6 +45,8 @@ automatically respawned.
 * QUIT - Gracefully exit after finishing the current request.
   Unless WINCH has been sent to the master (or the master is killed),
   the master process will respawn a worker to replace this one.
+  This currently does not wait for requests deferred to a separate
+  thread when using EventMachine (when app.deferred?(env) => true)
 
 * USR1 - Reopen all logs owned by the worker process.
   See Unicorn::Util.reopen_logs for what is considered a log.

data/TODO

@@ -10,12 +10,9 @@ care about.
   unit tests, only integration tests that exercise externally
   visible parts.
 
-* EventMachine.spawn - should be like Revactor, maybe?
-
-* conditional app.deferred?(env) support
-  Merb uses it, some other servers support it
-
 * EventMachine+Fibers+streaming input
+  (those who do not require streaming input can use
+  {rack-fiber_pool}[http://github.com/mperham/rack-fiber_pool])
 
 * RevFiberPool
 

data/config/.gitignore

@@ -0,0 +1 @@
+/isolate_*.rb

data/config/isolate.rb

@@ -0,0 +1,25 @@
+# this the default config file used by John Barnette's isolate gem
+# you can create a config/isolate_local.rb file to override this
+# See the corresponding tasks in Rakefile and GNUmakefile
+# `rake isolate' or (faster in the unmodified case, `make isolate')
+
+gem 'rack', '1.1.0'
+gem 'unicorn', '0.97.1'
+
+gem 'iobuffer', '0.1.3'
+gem 'rev', '0.3.2'
+
+gem 'eventmachine', '0.12.10'
+
+gem 'sinatra', '0.9.4'
+gem 'async_sinatra', '0.1.5'
+
+gem 'neverblock', '0.1.6.2'
+
+if defined?(::Fiber)
+  gem 'case', '0.5'
+  gem 'revactor', '0.1.5'
+  gem 'rack-fiber_pool', '0.9.0'
+end
+
+gem 'cramp', '0.10'

data/lib/rainbows/base.rb

@@ -12,6 +12,7 @@ module Rainbows
 
     def init_worker_process(worker)
       super(worker)
+      MaxBody.setup
       G.tmp = worker.tmp
 
       # avoid spurious wakeups and blocking-accept() with 1.8 green threads
@@ -29,14 +30,35 @@ module Rainbows
       logger.info "Rainbows! #@use worker_connections=#@worker_connections"
     end
 
+    if IO.respond_to?(:copy_stream)
+      def write_body(client, body)
+        if body.respond_to?(:to_path)
+          io = body.respond_to?(:to_io) ? body.to_io : body.to_path
+          IO.copy_stream(io, client)
+        else
+          body.each { |chunk| client.write(chunk) }
+        end
+        ensure
+          body.respond_to?(:close) and body.close
+      end
+    else
+      def write_body(client, body)
+        body.each { |chunk| client.write(chunk) }
+        ensure
+          body.respond_to?(:close) and body.close
+      end
+    end
+
     # once a client is accepted, it is processed in its entirety here
     # in 3 easy steps: read request, call app, write app response
+    # this is used by synchronous concurrency models
+    #   Base, ThreadSpawn, ThreadPool
     def process_client(client)
       buf = client.readpartial(CHUNK_SIZE) # accept filters protect us here
       hp = HttpParser.new
       env = {}
       alive = true
-      remote_addr = TCPSocket === client ? client.peeraddr.last : LOCALHOST
+      remote_addr = Rainbows.addr(client)
 
       begin # loop
         while ! hp.headers(env, buf)
@@ -46,20 +68,22 @@ module Rainbows
 
         env[CLIENT_IO] = client
         env[RACK_INPUT] = 0 == hp.content_length ?
-                 HttpRequest::NULL_IO :
-                 Unicorn::TeeInput.new(client, env, hp, buf)
+                 HttpRequest::NULL_IO : TeeInput.new(client, env, hp, buf)
         env[REMOTE_ADDR] = remote_addr
-        response = app.call(env.update(RACK_DEFAULTS))
+        status, headers, body = app.call(env.update(RACK_DEFAULTS))
 
-        if 100 == response.first.to_i
+        if 100 == status.to_i
           client.write(EXPECT_100_RESPONSE)
           env.delete(HTTP_EXPECT)
-          response = app.call(env)
+          status, headers, body = app.call(env)
         end
 
         alive = hp.keepalive? && G.alive
-        out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
-        HttpResponse.write(client, response, out)
+        if hp.headers?
+          out = [ alive ? CONN_ALIVE : CONN_CLOSE ]
+          client.write(HttpResponse.header_string(status, headers, out))
+        end
+        write_body(client, body)
       end while alive and hp.reset.nil? and env.clear
     # if we get any error, try to write something back to the client
     # assuming we haven't closed the socket, but don't get hung up

data/lib/rainbows/const.rb

@@ -3,7 +3,7 @@
 module Rainbows
 
   module Const
-    RAINBOWS_VERSION = '0.91.1'
+    RAINBOWS_VERSION = '0.92.0'
 
     include Unicorn::Const
 
@@ -17,7 +17,6 @@ module Rainbows
 
     CONN_CLOSE = "Connection: close\r\n"
     CONN_ALIVE = "Connection: keep-alive\r\n"
-    LOCALHOST = Unicorn::HttpRequest::LOCALHOST
 
     # client IO object that supports reading and writing directly
     # without filtering it through the HTTP chunk parser.
@@ -25,5 +24,7 @@ module Rainbows
     # of the official spec, but for now it is "hack.io"
     CLIENT_IO = "hack.io".freeze
 
+    ERROR_413_RESPONSE = "HTTP/1.1 413 Request Entity Too Large\r\n\r\n"
+
   end
 end

data/lib/rainbows/ev_core.rb

@@ -14,7 +14,7 @@ module Rainbows
     ASYNC_CLOSE = "async.close".freeze
 
     def post_init
-      @remote_addr = ::TCPSocket === @_io ? @_io.peeraddr.last : LOCALHOST
+      @remote_addr = Rainbows.addr(@_io)
       @env = {}
       @hp = HttpParser.new
       @state = :headers # [ :body [ :trailers ] ] :app_call :close
@@ -49,7 +49,7 @@ module Rainbows
             write(EXPECT_100_RESPONSE)
             @env.delete(HTTP_EXPECT)
           end
-          @input = len && len <= MAX_BODY ? StringIO.new("") : Util.tmpio
+          @input = CapInput.new(len, self)
           @hp.filter_body(@buf2 = "", @buf)
           @input << @buf2
           on_read("")
@@ -73,5 +73,45 @@ module Rainbows
         handle_error(e)
     end
 
+    class CapInput < Struct.new(:io, :client, :bytes_left)
+      MAX_BODY = Unicorn::Const::MAX_BODY
+      Util = Unicorn::Util
+
+      def self.err(client, msg)
+        client.write(Const::ERROR_413_RESPONSE)
+        client.quit
+
+        # zip back up the stack
+        raise IOError, msg, []
+      end
+
+      def self.new(len, client)
+        max = Rainbows.max_bytes
+        if len
+          if max && (len > max)
+            err(client, "Content-Length too big: #{len} > #{max}")
+          end
+          len <= MAX_BODY ? StringIO.new("") : Util.tmpio
+        else
+          max ? super(Util.tmpio, client, max) : Util.tmpio
+        end
+      end
+
+      def <<(buf)
+        if (self.bytes_left -= buf.size) < 0
+          io.close
+          CapInput.err(client, "chunked request body too big")
+        end
+        io << buf
+      end
+
+      def gets; io.gets; end
+      def each(&block); io.each(&block); end
+      def size; io.size; end
+      def rewind; io.rewind; end
+      def read(*args); io.read(*args); end
+
+    end
+
   end
 end

data/lib/rainbows/event_machine.rb

@@ -8,11 +8,14 @@ module Rainbows
   # Implements a basic single-threaded event model with
   # {EventMachine}[http://rubyeventmachine.com/].  It is capable of
   # handling thousands of simultaneous client connections, but with only
-  # a single-threaded app dispatch.  It is suited for slow clients and
-  # fast applications (applications that do not have slow network
-  # dependencies) or applications that use DevFdResponse for deferrable
-  # response bodies.  It does not require your Rack application to be
-  # thread-safe, reentrancy is only required for the DevFdResponse body
+  # a single-threaded app dispatch.  It is suited for slow clients,
+  # and can work with slow applications via asynchronous libraries such as
+  # {async_sinatra}[http://github.com/raggi/async_sinatra],
+  # {Cramp}[http://m.onkey.org/2010/1/7/introducing-cramp],
+  # and {rack-fiber_pool}[http://github.com/mperham/rack-fiber_pool].
+  #
+  # It does not require your Rack application to be thread-safe,
+  # reentrancy is only required for the DevFdResponse body
   # generator.
   #
   # Compatibility: Whatever \EventMachine ~> 0.12.10 and Unicorn both
@@ -22,6 +25,20 @@ module Rainbows
   # environment such as
   # {async_sinatra}[http://github.com/raggi/async_sinatra].
   #
+  # For a complete asynchronous framework,
+  # {Cramp}[http://m.onkey.org/2010/1/7/introducing-cramp] is fully
+  # supported when using this concurrency model.
+  #
+  # This model is fully-compatible with
+  # {rack-fiber_pool}[http://github.com/mperham/rack-fiber_pool]
+  # which allows each request to run inside its own \Fiber after
+  # all request processing is complete.
+  #
+  # Merb (and other frameworks/apps) supporting +deferred?+ execution as
+  # documented at http://brainspl.at/articles/2008/04/18/deferred-requests-with-merb-ebb-and-thin
+  # will also get the ability to conditionally defer request processing
+  # to a separate thread.
+  #
   # This model does not implement as streaming "rack.input" which allows
   # the Rack application to process data as it arrives.  This means
   # "rack.input" will be fully buffered in memory or to a temporary file
@@ -188,11 +205,37 @@ module Rainbows
       end
     end
 
+    # Middleware that will run the app dispatch in a separate thread.
+    # This middleware is automatically loaded by Rainbows! when using
+    # EventMachine and if the app responds to the +deferred?+ method.
+    class TryDefer < Struct.new(:app)
+
+      def initialize(app)
+        # the entire app becomes multithreaded, even the root (non-deferred)
+        # thread since any thread can share processes with others
+        Const::RACK_DEFAULTS['rack.multithread'] = true
+        super
+      end
+
+      def call(env)
+        if app.deferred?(env)
+          EM.defer(proc { catch(:async) { app.call(env) } },
+                   env[EvCore::ASYNC_CALLBACK])
+          # all of the async/deferred stuff breaks Rack::Lint :<
+          nil
+        else
+          app.call(env)
+        end
+      end
+    end
+
     # runs inside each forked worker, this sits around and waits
     # for connections and doesn't die until the parent dies (or is
     # given a INT, QUIT, or TERM signal)
     def worker_loop(worker)
       init_worker_process(worker)
+      G.server.app.respond_to?(:deferred?) and
+        G.server.app = TryDefer[G.server.app]
 
       # enable them both, should be non-fatal if not supported
       EM.epoll

data/lib/rainbows/fiber/base.rb

@@ -57,15 +57,17 @@ module Rainbows
       def schedule_sleepers
         max = nil
         now = Time.now
+        fibs = []
         ZZ.delete_if { |fib, time|
           if now >= time
-            fib.resume
+            fibs << fib
             now = Time.now
           else
             max = time
             false
           end
         }
+        fibs.each { |fib| fib.resume }
         max.nil? || max > (now + 1) ? 1 : max - now
       end
 
@@ -76,7 +78,7 @@ module Rainbows
         hp = HttpParser.new
         env = {}
         alive = true
-        remote_addr = TCPSocket === io ? io.peeraddr.last : LOCALHOST
+        remote_addr = Rainbows.addr(io)
 
         begin # loop
           while ! hp.headers(env, buf)

data/lib/rainbows/fiber/rev.rb

@@ -80,7 +80,7 @@ module Rainbows::Fiber
         hp = HttpParser.new
         env = {}
         alive = true
-        remote_addr = TCPSocket === io ? io.peeraddr.last : LOCALHOST
+        remote_addr = Rainbows.addr(io)
 
         begin # loop
           buf << (client.read_timeout or return) until hp.headers(env, buf)

data/lib/rainbows/http_response.rb

@@ -1,34 +1,35 @@
 # -*- encoding: binary -*-
 require 'time'
-require 'rainbows'
 
 module Rainbows
 
   class HttpResponse < ::Unicorn::HttpResponse
 
-    def self.write(socket, rack_response, out = [])
-      status, headers, body = rack_response
-
-      if Array === out
-        status = CODES[status.to_i] || status
+    def self.header_string(status, headers, out)
+      status = CODES[status.to_i] || status
 
-        headers.each do |key, value|
-          next if %r{\AX-Rainbows-}i =~ key
-          next if SKIP.include?(key.downcase)
-          if value =~ /\n/
-            # avoiding blank, key-only cookies with /\n+/
-            out.concat(value.split(/\n+/).map! { |v| "#{key}: #{v}\r\n" })
-          else
-            out << "#{key}: #{value}\r\n"
-          end
+      headers.each do |key, value|
+        next if %r{\AX-Rainbows-}i =~ key
+        next if SKIP.include?(key.downcase)
+        if value =~ /\n/
+          # avoiding blank, key-only cookies with /\n+/
+          out.concat(value.split(/\n+/).map! { |v| "#{key}: #{v}\r\n" })
+        else
+          out << "#{key}: #{value}\r\n"
         end
-
-        socket.write("HTTP/1.1 #{status}\r\n" \
-                     "Date: #{Time.now.httpdate}\r\n" \
-                     "Status: #{status}\r\n" \
-                     "#{out.join('')}\r\n")
       end
 
+      "HTTP/1.1 #{status}\r\n" \
+      "Date: #{Time.now.httpdate}\r\n" \
+      "Status: #{status}\r\n" \
+      "#{out.join('')}\r\n"
+    end
+
+    def self.write(socket, rack_response, out = [])
+      status, headers, body = rack_response
+      out.instance_of?(Array) and
+        socket.write(header_string(status, headers, out))
+
       body.each { |chunk| socket.write(chunk) }
       ensure
         body.respond_to?(:close) and body.close

data/lib/rainbows/http_server.rb

@@ -1,5 +1,4 @@
 # -*- encoding: binary -*-
-require 'rainbows'
 module Rainbows
 
   class HttpServer < ::Unicorn::HttpServer
@@ -61,11 +60,7 @@ module Rainbows
       end
       mod.setup if mod.respond_to?(:setup)
       Const::RACK_DEFAULTS['rainbows.model'] = @use = model.to_sym
-
-      Const::RACK_DEFAULTS['rack.multithread'] = case model.to_s
-      when /Thread/, "EventMachineDefer"; true
-      else false
-      end
+      Const::RACK_DEFAULTS['rack.multithread'] = !!(model.to_s =~ /Thread/)
 
       case @use
       when :Rev, :EventMachine, :NeverBlock
@@ -86,6 +81,18 @@ module Rainbows
         raise ArgumentError, "keepalive must be a non-negative Integer"
       G.kato = nr
     end
+
+    def client_max_body_size(nr)
+      err = "client_max_body_size must be nil or a non-negative Integer"
+      case nr
+      when nil
+      when Integer
+        nr >= 0 or raise ArgumentError, err
+      else
+        raise ArgumentError, err
+      end
+      Rainbows.max_bytes = nr
+    end
   end
 
 end