rainbows 0.95.1 → 0.96.0

data/Documentation/comparison.haml

@@ -89,7 +89,7 @@
     %td.mod RevThreadPool
     %td.tee No
     %td.r18 Yes
-    %td.r19 Yes
+    %td.r19 No
     %td.rbx No
     %td.slow Yes
   %tr.comp_row

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.95.1.GIT
+DEF_VER=v0.96.0.GIT
 
 LF='
 '

data/GNUmakefile

@@ -62,7 +62,7 @@ NEWS: GIT-VERSION-FILE
 	$(RAKE) -s news_rdoc > $@+
 	mv $@+ $@
 
-SINCE = 0.94.0
+SINCE = 0.95.0
 ChangeLog: LOG_VERSION = \
   $(shell git rev-parse -q "$(GIT_VERSION)" >/dev/null 2>&1 && \
           echo $(GIT_VERSION) || git describe)
@@ -183,6 +183,7 @@ release: verify package $(release_notes) $(release_changes)
 	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
 	$(RAKE) raa_update VERSION=$(VERSION)
 	$(RAKE) fm_update VERSION=$(VERSION)
+	$(RAKE) publish_news VERSION=$(VERSION)
 else
 gem install-gem: GIT-VERSION-FILE
 	$(MAKE) $@ VERSION=$(GIT_VERSION)

data/Rakefile

@@ -1,5 +1,6 @@
 # -*- encoding: binary -*-
 autoload :Gem, 'rubygems'
+autoload :Tempfile, 'tempfile'
 
 # most tasks are in the GNUmakefile which offers better parallelism
 
@@ -105,8 +106,21 @@ end
 desc "read news article from STDIN and post to rubyforge"
 task :publish_news do
   require 'rubyforge'
-  IO.select([STDIN], nil, nil, 1) or abort "E: news must be read from stdin"
-  msg = STDIN.readlines
+  spec = Gem::Specification.load('rainbows.gemspec')
+  tmp = Tempfile.new('rf-news')
+  _, subject, body = `git cat-file tag v#{spec.version}`.split(/\n\n/, 3)
+  tmp.puts subject
+  tmp.puts
+  tmp.puts spec.description.strip
+  tmp.puts ""
+  tmp.puts "* #{spec.homepage}"
+  tmp.puts "* #{spec.email}"
+  tmp.puts "* #{git_url}"
+  tmp.print "\nChanges:\n\n"
+  tmp.puts body
+  tmp.flush
+  system(ENV["VISUAL"], tmp.path) or abort "#{ENV["VISUAL"]} failed: #$?"
+  msg = File.readlines(tmp.path)
   subject = msg.shift
   blank = msg.shift
   blank == "\n" or abort "no newline after subject!"
@@ -157,7 +171,6 @@ end
 
 desc "post to FM"
 task :fm_update do
-  require 'tempfile'
   require 'net/http'
   require 'net/netrc'
   require 'json'

data/Static_Files

@@ -10,7 +10,7 @@ to simplify your deployments and only deploy one server?
 == {sendfile}[http://rubygems.org/gems/sendfile] RubyGem
 
 To enable the "sendfile" gem, just make sure you have 1.0.0 or later and
-"require" it in your Rainbows!/Unicorn config file (not your Rack
+"require" it in your \Rainbows!/Unicorn config file (not your Rack
 config.ru):
 
     require 'sendfile' # that's it! nothing else to do
@@ -24,9 +24,9 @@ config.ru):
     end
 
 The sendfile gem is works for all of our concurrency models except
-Revactor, NeverBlock and EventMachine (see below).
+NeverBlock and EventMachine (see below).
 
-The sendfile gem is less buggy than current (Ruby 1.9.2-rc1)
+The sendfile gem is less buggy than current (Ruby 1.9.2-rc2)
 IO.copy_stream and supports FreeBSD and Solaris in addition to Linux.
 This RubyGem also works under Ruby 1.8 (even with threads) and should
 work with rubinius.git, too.
@@ -40,17 +40,18 @@ their Writer* variants use the core IO.copy_stream method under Ruby
 1.9.  IO.copy_stream uses sendfile() under Linux, and a pread()/write()
 loop (implemented in C) on other systems.
 
-IO.copy_stream under Linux with Ruby 1.9.2-rc1 (and before) is also
+IO.copy_stream under Linux with Ruby 1.9.3 (and before) is also
 subject to hanging indefinitely when a client disconnected prematurely.
-This issue is fixed in Ruby trunk (July 2010) and will be in the next
-Ruby 1.9.2 release.
+This issue is fixed in Ruby trunk (July 2010).
 
 \Rainbows! supports IO.copy_stream since v0.93.0
 
 == EventMachine FileStreamer
 
-EventMachine and NeverBlock users automatically take advantage of
-the mmap()-based FileStreamer class distributed with EventMachine.
+EventMachine and NeverBlock users automatically take advantage of the
+mmap()-based FileStreamer class distributed with EventMachine.
+Unfortunately, as of EventMachine 0.12.10, FileStreamer cannot easily
+support HTTP Range responses.
 
 \Rainbows! supports EventMachine FileStreamer since v0.4.0
 
@@ -63,9 +64,8 @@ slower clients and smaller files.
 
 == The Future...
 
-Future releases of \Rainbows! will have byte-range support to serve
-partial and multipart responses, too.  We'll also support an open file
-cache (similar to nginx) which allows us to reuse open file descriptors.
+We'll also support an open file cache (similar to nginx) which
+allows us to reuse open file descriptors.
 
 Under Linux, we'll support the splice(2) system call for zero-copy
 proxying {io_splice}[http://bogomips.org/ruby_io_splice/], too.

data/TODO

@@ -11,13 +11,11 @@ care about.
 * allow _OPTIONAL_ splice(2) with DevFdResponse under Linux
   (splice is very broken under some older kernels)
 
-* use IO#sendfile_nonblock for EventMachine/Revactor/NeverBlock
+* use IO#sendfile_nonblock for EventMachine/NeverBlock
 
 * Open file cache Rack app/middleware (idea from nginx), since sendfile
   (and IO.copy_stream) allows pread(2)-style offsets
 
-* Byte range responses for static files + sendfile
-
 * Improve test suite coverage.  We won't waste cycles with puny
   unit tests, only integration tests that exercise externally
   visible parts.
@@ -36,7 +34,4 @@ care about.
 
 * Packet - pure Ruby, EventMachine-like library
 
-* Rubinius Actors - should be like Revactor and easily doable once
-  Rubinius gets more mature.
-
 * test and improve performance (throughput/latency/memory usage)

data/lib/rainbows.rb

@@ -29,13 +29,14 @@ module Rainbows
   end
   G = State.new(true, 0, 0, 5)
   O = {}
+  class Response416 < RangeError; end
   # :startdoc:
 
   require 'rainbows/const'
   require 'rainbows/http_server'
   require 'rainbows/response'
-  require 'rainbows/base'
-  require 'rainbows/tee_input'
+  autoload :Base, 'rainbows/base'
+  autoload :TeeInput, 'rainbows/tee_input'
   autoload :Sendfile, 'rainbows/sendfile'
   autoload :AppPool, 'rainbows/app_pool'
   autoload :DevFdResponse, 'rainbows/dev_fd_response'

data/lib/rainbows/base.rb

@@ -1,5 +1,4 @@
 # -*- encoding: binary -*-
-require 'rainbows/tee_input'
 
 # base class for Rainbows concurrency models, this is currently used by
 # ThreadSpawn and ThreadPool models.  Base is also its own
@@ -52,7 +51,6 @@ module Rainbows::Base
     buf = client.readpartial(CHUNK_SIZE) # accept filters protect us here
     hp = HttpParser.new
     env = {}
-    alive = true
     remote_addr = Rainbows.addr(client)
 
     begin # loop
@@ -65,18 +63,23 @@ module Rainbows::Base
       env[RACK_INPUT] = 0 == hp.content_length ?
                         NULL_IO : TeeInput.new(client, env, hp, buf)
       env[REMOTE_ADDR] = remote_addr
-      response = app.call(env.update(RACK_DEFAULTS))
+      status, headers, body = app.call(env.update(RACK_DEFAULTS))
 
-      if 100 == response[0].to_i
+      if 100 == status.to_i
         client.write(EXPECT_100_RESPONSE)
         env.delete(HTTP_EXPECT)
-        response = app.call(env)
+        status, headers, body = app.call(env)
       end
 
-      alive = hp.keepalive? && G.alive
-      out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
-      write_response(client, response, out)
-    end while alive and hp.reset.nil? and env.clear
+      if hp.headers?
+        headers = HH.new(headers)
+        range = make_range!(env, status, headers) and status = range.shift
+        env = false unless hp.keepalive? && G.alive
+        headers[CONNECTION] = env ? KEEP_ALIVE : CLOSE
+        client.write(response_header(status, headers))
+      end
+      write_body(client, body, range)
+    end while env && env.clear && hp.reset.nil?
   # if we get any error, try to write something back to the client
   # assuming we haven't closed the socket, but don't get hung up
   # if the socket is already closed or broken.  We'll always ensure

data/lib/rainbows/const.rb

@@ -3,7 +3,7 @@
 module Rainbows
 
   module Const
-    RAINBOWS_VERSION = '0.95.1'
+    RAINBOWS_VERSION = '0.96.0'
 
     include Unicorn::Const
 
@@ -15,9 +15,6 @@ module Rainbows
       # "rainbows.autochunk" => false,
     })
 
-    CONN_CLOSE = "Connection: close\r\n"
-    CONN_ALIVE = "Connection: keep-alive\r\n"
-
     # client IO object that supports reading and writing directly
     # without filtering it through the HTTP chunk parser.
     # Maybe we can get this renamed to "rack.io" if it becomes part
@@ -25,6 +22,7 @@ module Rainbows
     CLIENT_IO = "hack.io".freeze
 
     ERROR_413_RESPONSE = "HTTP/1.1 413 Request Entity Too Large\r\n\r\n"
+    ERROR_416_RESPONSE = "HTTP/1.1 416 Requested Range Not Satisfiable\r\n\r\n"
 
   end
 end

data/lib/rainbows/dev_fd_response.rb

@@ -31,34 +31,38 @@ class Rainbows::DevFdResponse < Struct.new(:app)
     return response if STATUS_WITH_NO_ENTITY_BODY.include?(status)
 
     io = body.to_io if body.respond_to?(:to_io)
-    io ||= File.open(body.to_path, 'rb') if body.respond_to?(:to_path)
+    io ||= File.open(body.to_path) if body.respond_to?(:to_path)
     return response if io.nil?
 
     headers = HeaderHash.new(headers)
     st = io.stat
+    fileno = io.fileno
     if st.file?
       headers['Content-Length'] ||= st.size.to_s
       headers.delete('Transfer-Encoding')
     elsif st.pipe? || st.socket? # epoll-able things
-      if env['rainbows.autochunk']
-        headers['Transfer-Encoding'] = 'chunked'
-        headers.delete('Content-Length')
-      else
-        headers['X-Rainbows-Autochunk'] = 'no'
+      unless headers['Content-Length']
+        if env['rainbows.autochunk']
+          headers['Transfer-Encoding'] = 'chunked'
+        else
+          headers['X-Rainbows-Autochunk'] = 'no'
+        end
       end
 
       # we need to make sure our pipe output is Fiber-compatible
       case env["rainbows.model"]
       when :FiberSpawn, :FiberPool, :RevFiberSpawn
-        return [ status, headers, Rainbows::Fiber::IO.new(io,::Fiber.current) ]
+        io = Rainbows::Fiber::IO.new(io,::Fiber.current)
+      when :Revactor
+        io = Rainbows::Revactor::Proxy.new(io)
       end
     else # unlikely, char/block device file, directory, ...
       return response
     end
-    [ status, headers, Body.new(io, "/dev/fd/#{io.fileno}") ]
+    [ status, headers, Body.new(io, "/dev/fd/#{fileno}", body) ]
   end
 
-  class Body < Struct.new(:to_io, :to_path)
+  class Body < Struct.new(:to_io, :to_path, :orig_body)
     # called by the webserver or other middlewares if they can't
     # handle #to_path
     def each(&block)
@@ -73,10 +77,9 @@ class Rainbows::DevFdResponse < Struct.new(:app)
 
     # called by the web server after #each
     def close
-      begin
-        to_io.close if to_io.respond_to?(:close)
-      rescue IOError # could've been IO::new()'ed and closed
-      end
+      to_io.close unless to_io.closed?
+      orig_body.close if orig_body.respond_to?(:close) # may not be an IO
+    rescue IOError # could've been IO::new()'ed and closed
     end
   end
   #:startdoc:

data/lib/rainbows/error.rb

@@ -32,6 +32,8 @@ module Rainbows
         when EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EINVAL,
              Errno::EBADF, Errno::ENOTCONN
           # swallow error if client shuts down one end or disconnects
+        when Rainbows::Response416
+          Const::ERROR_416_RESPONSE
         when Unicorn::HttpParserError
           Const::ERROR_400_RESPONSE # try to tell the client they're bad
         when IOError # HttpParserError is an IOError

data/lib/rainbows/ev_core.rb

@@ -6,6 +6,7 @@ module Rainbows
   module EvCore
     include Unicorn
     include Rainbows::Const
+    include Rainbows::Response
     G = Rainbows::G
     NULL_IO = Unicorn::HttpRequest::NULL_IO
 
@@ -33,6 +34,18 @@ module Rainbows
         quit
     end
 
+    # returns whether to enable response chunking for autochunk models
+    def stream_response_headers(status, headers)
+      if headers['Content-Length']
+        rv = false
+      else
+        rv = !!(headers['Transfer-Encoding'] =~ %r{\Achunked\z}i)
+        rv = false if headers.delete('X-Rainbows-Autochunk') == 'no'
+      end
+      write(response_header(status, headers))
+      rv
+    end
+
     # TeeInput doesn't map too well to this right now...
     def on_read(data)
       case @state

data/lib/rainbows/event_machine.rb

@@ -47,10 +47,13 @@ module Rainbows
   module EventMachine
 
     include Base
+    autoload :ResponsePipe, 'rainbows/event_machine/response_pipe'
+    autoload :ResponseChunkPipe, 'rainbows/event_machine/response_chunk_pipe'
+    autoload :TryDefer, 'rainbows/event_machine/try_defer'
 
     class Client < EM::Connection # :nodoc: all
+      attr_writer :body
       include Rainbows::EvCore
-      include Rainbows::Response
       G = Rainbows::G
 
       def initialize(io)
@@ -59,7 +62,19 @@ module Rainbows
       end
 
       alias write send_data
-      alias receive_data on_read
+
+      def receive_data(data)
+        # To avoid clobbering the current streaming response
+        # (often a static file), we do not attempt to process another
+        # request on the same connection until the first is complete
+        if @body
+          @buf << data
+          @_io.shutdown(Socket::SHUT_RD) if @buf.size > 0x1c000
+          return EM.next_tick { receive_data('') }
+        else
+          on_read(data)
+        end
+      end
 
       def quit
         super
@@ -68,122 +83,88 @@ module Rainbows
 
       def app_call
         set_comm_inactivity_timeout 0
-        begin
-          @env[RACK_INPUT] = @input
-          @env[REMOTE_ADDR] = @remote_addr
-          @env[ASYNC_CALLBACK] = method(:em_write_response)
-
-          # we're not sure if anybody uses this, but Thin sets it, too
-          @env[ASYNC_CLOSE] = EM::DefaultDeferrable.new
-
-          response = catch(:async) { APP.call(@env.update(RACK_DEFAULTS)) }
-
-          # too tricky to support pipelining with :async since the
-          # second (pipelined) request could be a stuck behind a
-          # long-running async response
-          (response.nil? || -1 == response[0]) and return @state = :close
-
-          alive = @hp.keepalive? && G.alive
-          out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if @hp.headers?
-          em_write_response(response, out, alive)
-
-          if alive
-            @env.clear
-            @hp.reset
-            @state = :headers
-            # keepalive requests are always body-less, so @input is unchanged
-            @hp.headers(@env, @buf) and next
-            set_comm_inactivity_timeout G.kato
+        @env[RACK_INPUT] = @input
+        @env[REMOTE_ADDR] = @remote_addr
+        @env[ASYNC_CALLBACK] = method(:em_write_response)
+        @env[ASYNC_CLOSE] = EM::DefaultDeferrable.new
+
+        response = catch(:async) { APP.call(@env.update(RACK_DEFAULTS)) }
+
+        # too tricky to support pipelining with :async since the
+        # second (pipelined) request could be a stuck behind a
+        # long-running async response
+        (response.nil? || -1 == response[0]) and return @state = :close
+
+        em_write_response(response, alive = @hp.keepalive? && G.alive)
+        if alive
+          @env.clear
+          @hp.reset
+          @state = :headers
+          if @buf.empty?
+            set_comm_inactivity_timeout(G.kato)
+          else
+            EM.next_tick { receive_data('') }
           end
-          return
-        end while true
+        end
       end
 
-      def em_write_response(response, out = [ CONN_CLOSE ], alive = false)
-        @body = body = response[2]
+      def em_write_response(response, alive = false)
+        status, headers, body = response
+        if @hp.headers?
+          headers = HH.new(headers)
+          headers[CONNECTION] = alive ? KEEP_ALIVE : CLOSE
+        else
+          headers = nil
+        end
+
         if body.respond_to?(:errback) && body.respond_to?(:callback)
+          @body = body
           body.callback { quit }
           body.errback { quit }
-          write_header(self, response, out)
-          write_body_each(self, body)
-          return
-        elsif ! body.respond_to?(:to_path)
-          write_response(self, response, out)
-          quit unless alive
-          return
-        end
-
-        headers = Rack::Utils::HeaderHash.new(response[1])
-        io = body_to_io(body)
-        st = io.stat
-
-        if st.file?
-          headers.delete('Transfer-Encoding')
-          headers['Content-Length'] ||= st.size.to_s
-          write_header(self, [ response[0], headers ], out)
-          stream = stream_file_data(body.to_path)
-          stream.callback { quit } unless alive
-        elsif st.socket? || st.pipe?
-          do_chunk = !!(headers['Transfer-Encoding'] =~ %r{\Achunked\z}i)
-          do_chunk = false if headers.delete('X-Rainbows-Autochunk') == 'no'
-          if out.nil?
-            do_chunk = false
-          else
-            out[0] = CONN_CLOSE
-          end
-          write_header(self, [ response[0], headers ], out)
-          if do_chunk
-            EM.watch(io, ResponseChunkPipe, self).notify_readable = true
-          else
-            EM.enable_proxy(EM.attach(io, ResponsePipe, self), self, 16384)
+          # async response, this could be a trickle as is in comet-style apps
+          headers[CONNECTION] = CLOSE if headers
+          alive = true
+        elsif body.respond_to?(:to_path)
+          st = File.stat(path = body.to_path)
+
+          if st.file?
+            write(response_header(status, headers)) if headers
+            @body = stream_file_data(path)
+            @body.errback do
+              body.close if body.respond_to?(:close)
+              quit
+            end
+            @body.callback do
+              body.close if body.respond_to?(:close)
+              @body = nil
+              alive ? receive_data('') : quit
+            end
+            return
+          elsif st.socket? || st.pipe?
+            @body = io = body_to_io(body)
+            chunk = stream_response_headers(status, headers) if headers
+            m = chunk ? ResponseChunkPipe : ResponsePipe
+            return EM.watch(io, m, self, alive, body).notify_readable = true
           end
-        else
-          write_response(self, response, out)
+          # char or block device... WTF? fall through to body.each
         end
+
+        write(response_header(status, headers)) if headers
+        write_body_each(self, body)
+        quit unless alive
       end
 
       def unbind
         async_close = @env[ASYNC_CLOSE] and async_close.succeed
         @body.respond_to?(:fail) and @body.fail
-        @_io.close
-      end
-    end
-
-    module ResponsePipe # :nodoc: all
-      def initialize(client)
-        @client = client
-      end
-
-      def unbind
-        @io.close
-        @client.quit
-      end
-    end
-
-    module ResponseChunkPipe # :nodoc: all
-      include ResponsePipe
-
-      def unbind
-        @client.write("0\r\n\r\n")
-        super
-      end
-
-      def notify_readable
         begin
-          data = begin
-            @io.read_nonblock(16384)
-          rescue Errno::EINTR
-            retry
-          rescue Errno::EAGAIN
-            return
-          rescue EOFError
-            detach
-            return
-          end
-          @client.send_data(sprintf("%x\r\n", data.size))
-          @client.send_data(data)
-          @client.send_data("\r\n")
-        end while true
+          @_io.close
+        rescue Errno::EBADF
+          # EventMachine's EventableDescriptor::Close() may close
+          # the underlying file descriptor without invalidating the
+          # associated IO object on errors, so @_io.closed? isn't
+          # sufficient.
+        end
       end
     end
 
@@ -202,30 +183,6 @@ module Rainbows
       end
     end
 
-    # Middleware that will run the app dispatch in a separate thread.
-    # This middleware is automatically loaded by Rainbows! when using
-    # EventMachine and if the app responds to the +deferred?+ method.
-    class TryDefer < Struct.new(:app) # :nodoc: all
-
-      def initialize(app)
-        # the entire app becomes multithreaded, even the root (non-deferred)
-        # thread since any thread can share processes with others
-        Const::RACK_DEFAULTS['rack.multithread'] = true
-        super
-      end
-
-      def call(env)
-        if app.deferred?(env)
-          EM.defer(proc { catch(:async) { app.call(env) } },
-                   env[EvCore::ASYNC_CALLBACK])
-          # all of the async/deferred stuff breaks Rack::Lint :<
-          nil
-        else
-          app.call(env)
-        end
-      end
-    end
-
     def init_worker_process(worker) # :nodoc:
       Rainbows::Response.setup(Rainbows::EventMachine::Client)
       super