rainbows 0.94.0 → 0.95.0

data/lib/rainbows/revactor.rb

@@ -2,174 +2,174 @@
 require 'revactor'
 Revactor::VERSION >= '0.1.5' or abort 'revactor 0.1.5 is required'
 
-module Rainbows
-
-  # Enables use of the Actor model through
-  # {Revactor}[http://revactor.org] under Ruby 1.9.  It spawns one
-  # long-lived Actor for every listen socket in the process and spawns a
-  # new Actor for every client connection accept()-ed.
-  # +worker_connections+ will limit the number of client Actors we have
-  # running at any one time.
-  #
-  # Applications using this model are required to be reentrant, but do
-  # not have to worry about race conditions unless they use threads
-  # internally.  \Rainbows! does not spawn threads under this model.
-  # Multiple instances of the same app may run in the same address space
-  # sequentially (but at interleaved points).  Any network dependencies
-  # in the application using this model should be implemented using the
-  # \Revactor library as well, to take advantage of the networking
-  # concurrency features this model provides.
-
-  module Revactor
-    RD_ARGS = {}
-
-    include Base
-
-    # once a client is accepted, it is processed in its entirety here
-    # in 3 easy steps: read request, call app, write app response
-    def process_client(client)
-      io = client.instance_variable_get(:@_io)
-      io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
-      rd_args = [ nil ]
-      remote_addr = if ::Revactor::TCP::Socket === client
-        rd_args << RD_ARGS
-        client.remote_addr
-      else
-        Unicorn::HttpRequest::LOCALHOST
-      end
-      buf = client.read(*rd_args)
-      hp = HttpParser.new
-      env = {}
-      alive = true
-
-      begin
-        while ! hp.headers(env, buf)
-          buf << client.read(*rd_args)
-        end
-
-        env[Const::CLIENT_IO] = client
-        env[Const::RACK_INPUT] = 0 == hp.content_length ?
-                 HttpRequest::NULL_IO :
-                 TeeInput.new(PartialSocket.new(client), env, hp, buf)
-        env[Const::REMOTE_ADDR] = remote_addr
-        response = app.call(env.update(RACK_DEFAULTS))
-
-        if 100 == response.first.to_i
-          client.write(Const::EXPECT_100_RESPONSE)
-          env.delete(Const::HTTP_EXPECT)
-          response = app.call(env)
-        end
-
-        alive = hp.keepalive? && G.alive
-        out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
-        HttpResponse.write(client, response, out)
-      end while alive and hp.reset.nil? and env.clear
-    rescue ::Revactor::TCP::ReadError
-    rescue => e
-      Error.write(io, e)
-    ensure
-      client.close
+# Enables use of the Actor model through
+# {Revactor}[http://revactor.org] under Ruby 1.9.  It spawns one
+# long-lived Actor for every listen socket in the process and spawns a
+# new Actor for every client connection accept()-ed.
+# +worker_connections+ will limit the number of client Actors we have
+# running at any one time.
+#
+# Applications using this model are required to be reentrant, but do
+# not have to worry about race conditions unless they use threads
+# internally.  \Rainbows! does not spawn threads under this model.
+# Multiple instances of the same app may run in the same address space
+# sequentially (but at interleaved points).  Any network dependencies
+# in the application using this model should be implemented using the
+# \Revactor library as well, to take advantage of the networking
+# concurrency features this model provides.
+module Rainbows::Revactor
+
+  # :stopdoc:
+  RD_ARGS = {}
+
+  include Rainbows::Base
+  LOCALHOST = Unicorn::HttpRequest::LOCALHOST
+  TCP = ::Revactor::TCP::Socket
+
+  # once a client is accepted, it is processed in its entirety here
+  # in 3 easy steps: read request, call app, write app response
+  def process_client(client) # :nodoc:
+    io = client.instance_variable_get(:@_io)
+    io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+    rd_args = [ nil ]
+    remote_addr = if TCP === client
+      rd_args << RD_ARGS
+      client.remote_addr
+    else
+      LOCALHOST
     end
+    buf = client.read(*rd_args)
+    hp = HttpParser.new
+    env = {}
+    alive = true
+
+    begin
+      buf << client.read(*rd_args) until hp.headers(env, buf)
+
+      env[CLIENT_IO] = client
+      env[RACK_INPUT] = 0 == hp.content_length ?
+               NULL_IO : TeeInput.new(PartialSocket.new(client), env, hp, buf)
+      env[REMOTE_ADDR] = remote_addr
+      response = app.call(env.update(RACK_DEFAULTS))
+
+      if 100 == response[0].to_i
+        client.write(EXPECT_100_RESPONSE)
+        env.delete(HTTP_EXPECT)
+        response = app.call(env)
+      end
 
-    # runs inside each forked worker, this sits around and waits
-    # for connections and doesn't die until the parent dies (or is
-    # given a INT, QUIT, or TERM signal)
-    def worker_loop(worker)
-      init_worker_process(worker)
-      RD_ARGS[:timeout] = G.kato if G.kato > 0
-      nr = 0
-      limit = worker_connections
-      actor_exit = Case[:exit, Actor, Object]
-
-      revactorize_listeners.each do |l, close, accept|
-        Actor.spawn(l, close, accept) do |l, close, accept|
-          Actor.current.trap_exit = true
-          l.controller = l.instance_variable_set(:@receiver, Actor.current)
-          begin
-            while nr >= limit
-              l.disable if l.enabled?
-              logger.info "busy: clients=#{nr} >= limit=#{limit}"
-              Actor.receive do |f|
-                f.when(close) {}
-                f.when(actor_exit) { nr -= 1 }
-                f.after(0.01) {} # another listener could've gotten an exit
-              end
-            end
+      alive = hp.keepalive? && G.alive
+      out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
+      write_response(client, response, out)
+    end while alive and hp.reset.nil? and env.clear
+  rescue ::Revactor::TCP::ReadError
+  rescue => e
+    Rainbows::Error.write(io, e)
+  ensure
+    client.close
+  end
 
-            l.enable unless l.enabled?
+  # runs inside each forked worker, this sits around and waits
+  # for connections and doesn't die until the parent dies (or is
+  # given a INT, QUIT, or TERM signal)
+  def worker_loop(worker) #:nodoc:
+    init_worker_process(worker)
+    self.class.__send__(:alias_method, :write_body, :write_body_each)
+    RD_ARGS[:timeout] = G.kato if G.kato > 0
+    nr = 0
+    limit = worker_connections
+    actor_exit = Case[:exit, Actor, Object]
+
+    revactorize_listeners.each do |l, close, accept|
+      Actor.spawn(l, close, accept) do |l, close, accept|
+        Actor.current.trap_exit = true
+        l.controller = l.instance_variable_set(:@receiver, Actor.current)
+        begin
+          while nr >= limit
+            l.disable if l.enabled?
+            logger.info "busy: clients=#{nr} >= limit=#{limit}"
             Actor.receive do |f|
               f.when(close) {}
               f.when(actor_exit) { nr -= 1 }
-              f.when(accept) do |_, _, s|
-                nr += 1
-                Actor.spawn_link(s) { |c| process_client(c) }
-              end
+              f.after(0.01) {} # another listener could've gotten an exit
             end
-          rescue => e
-            Error.listen_loop(e)
-          end while G.alive
+          end
+
+          l.enable unless l.enabled?
           Actor.receive do |f|
             f.when(close) {}
             f.when(actor_exit) { nr -= 1 }
-          end while nr > 0
-        end
+            f.when(accept) do |_, _, s|
+              nr += 1
+              Actor.spawn_link(s) { |c| process_client(c) }
+            end
+          end
+        rescue => e
+          Rainbows::Error.listen_loop(e)
+        end while G.alive
+        Actor.receive do |f|
+          f.when(close) {}
+          f.when(actor_exit) { nr -= 1 }
+        end while nr > 0
       end
-
-      Actor.sleep 1 while G.tick || nr > 0
-      rescue Errno::EMFILE
-        # ignore, let another worker process take it
     end
 
-    def revactorize_listeners
-      LISTENERS.map do |s|
-        case s
-        when TCPServer
-          l = ::Revactor::TCP.listen(s, nil)
-          [ l, T[:tcp_closed, ::Revactor::TCP::Socket],
-            T[:tcp_connection, l, ::Revactor::TCP::Socket] ]
-        when UNIXServer
-          l = ::Revactor::UNIX.listen(s)
-          [ l, T[:unix_closed, ::Revactor::UNIX::Socket ],
-            T[:unix_connection, l, ::Revactor::UNIX::Socket] ]
-        end
+    Actor.sleep 1 while G.tick || nr > 0
+    rescue Errno::EMFILE
+      # ignore, let another worker process take it
+  end
+
+  def revactorize_listeners
+    LISTENERS.map do |s|
+      case s
+      when TCPServer
+        l = ::Revactor::TCP.listen(s, nil)
+        [ l, T[:tcp_closed, ::Revactor::TCP::Socket],
+          T[:tcp_connection, l, ::Revactor::TCP::Socket] ]
+      when UNIXServer
+        l = ::Revactor::UNIX.listen(s)
+        [ l, T[:unix_closed, ::Revactor::UNIX::Socket ],
+          T[:unix_connection, l, ::Revactor::UNIX::Socket] ]
       end
     end
+  end
 
-    # Revactor Sockets do not implement readpartial, so we emulate just
-    # enough to avoid mucking with TeeInput internals.  Fortunately
-    # this code is not heavily used so we can usually avoid the overhead
-    # of adding a userspace buffer.
-    class PartialSocket < Struct.new(:socket, :rbuf)
-      def initialize(socket)
-        # IO::Buffer is used internally by Rev which Revactor is based on
-        # so we'll always have it available
-        super(socket, IO::Buffer.new)
-      end
+  # Revactor Sockets do not implement readpartial, so we emulate just
+  # enough to avoid mucking with TeeInput internals.  Fortunately
+  # this code is not heavily used so we can usually avoid the overhead
+  # of adding a userspace buffer.
+  class PartialSocket < Struct.new(:socket, :rbuf)
+    def initialize(socket)
+      # IO::Buffer is used internally by Rev which Revactor is based on
+      # so we'll always have it available
+      super(socket, IO::Buffer.new)
+    end
 
-      # Revactor socket reads always return an unspecified amount,
-      # sometimes too much
-      def readpartial(length, dst = "")
-        return dst if length == 0
-        # always check and return from the userspace buffer first
-        rbuf.size > 0 and return dst.replace(rbuf.read(length))
+    # Revactor socket reads always return an unspecified amount,
+    # sometimes too much
+    def readpartial(length, dst = "")
+      return dst.replace("") if length == 0
 
-        # read off the socket since there was nothing in rbuf
-        tmp = socket.read
+      # always check and return from the userspace buffer first
+      rbuf.size > 0 and return dst.replace(rbuf.read(length))
 
-        # we didn't read too much, good, just return it straight back
-        # to avoid needlessly wasting memory bandwidth
-        tmp.size <= length and return dst.replace(tmp)
+      # read off the socket since there was nothing in rbuf
+      tmp = socket.read
 
-        # ugh, read returned too much, copy + reread to avoid slicing
-        rbuf << tmp[length, tmp.size]
-        dst.replace(tmp[0, length])
-      end
+      # we didn't read too much, good, just return it straight back
+      # to avoid needlessly wasting memory bandwidth
+      tmp.size <= length and return dst.replace(tmp)
 
-      # just proxy any remaining methods TeeInput may use
-      def close
-        socket.close
-      end
+      # ugh, read returned too much
+      rbuf << tmp[length, tmp.size]
+      dst.replace(tmp[0, length])
     end
 
+    # just proxy any remaining methods TeeInput may use
+    def close
+      socket.close
+    end
   end
+
+  # :startdoc:
 end

data/lib/rainbows/sendfile.rb

@@ -11,8 +11,8 @@ module Rainbows
 # efficiently using sendfile() or similar.  With multithreaded models
 # under Ruby 1.9, IO.copy_stream will be used.
 #
-# This middleware is the opposite of Rack::Contrib::Sendfile as it
-# reverses the effect of Rack::Contrib::Sendfile.  Unlike many Ruby
+# This middleware is the opposite of Rack::Sendfile as it
+# reverses the effect of Rack:::Sendfile.  Unlike many Ruby
 # web servers, some configurations of \Rainbows! are capable of
 # serving static files efficiently.
 #
@@ -57,37 +57,26 @@ class Sendfile < Struct.new(:app)
   # Body wrapper, this allows us to fall back gracefully to
   # +each+ in case a given concurrency model does not optimize
   # +to_path+ calls.
-  class Body < Struct.new(:to_io)
-
-    def initialize(path, headers)
-      # Rainbows! will try #to_io if #to_path exists to avoid unnecessary
-      # open() calls.
-      self.to_io = File.open(path, 'rb')
+  class Body < Struct.new(:to_path) # :nodoc: all
 
+    def self.new(path, headers)
       unless headers['Content-Length']
-        stat = to_io.stat
+        stat = File.stat(path)
         headers['Content-Length'] = stat.size.to_s if stat.file?
       end
-    end
-
-    def to_path
-      to_io.path
+      super(path)
     end
 
     # fallback in case our +to_path+ doesn't get handled for whatever reason
     def each(&block)
-      buf = ''
-      while to_io.read(0x4000, buf)
-        yield buf
+      File.open(to_path, 'rb') do |fp|
+        buf = ''
+        yield buf while fp.read(0x4000, buf)
       end
     end
-
-    def close
-      to_io.close
-    end
   end
 
-  def call(env)
+  def call(env) # :nodoc:
     status, headers, body = app.call(env)
     headers = HH.new(headers)
     if path = headers.delete('X-Sendfile')

data/lib/rainbows/server_token.rb

@@ -0,0 +1,39 @@
+# -*- encoding: binary -*-
+module Rainbows
+
+# An optional middleware to proudly display your usage of \Rainbows! in
+# the "Server:" response header.  This means you can help tell the world
+# you're using \Rainbows! and spread fun and joy all over the Internet!
+#
+#    ------ in your config.ru ------
+#    require 'rainbows/server_token'
+#    require 'rack/lobster'
+#    use Rainbows::ServerToken
+#    run Rack::Lobster.new
+#
+# If you're nervous about the exact version of \Rainbows! you're running,
+# then you can actually specify anything you want:
+#
+#    use Rainbows::ServerToken, "netcat 1.0"
+#
+
+class ServerToken < Struct.new(:app, :token)
+
+  # :stopdoc:
+  #
+  # Freeze constants as they're slightly faster when setting hashes
+  SERVER = "Server".freeze
+
+  def initialize(app, token = Const::RACK_DEFAULTS['SERVER_SOFTWARE'])
+    super
+  end
+
+  def call(env)
+    status, headers, body = app.call(env)
+    headers = Rack::Utils::HeaderHash.new(headers)
+    headers[SERVER] = token
+    [ status, headers, body ]
+  end
+  # :startdoc:
+end
+end

data/lib/rainbows/stream_file.rb

@@ -0,0 +1,14 @@
+# -*- encoding: binary -*-
+# :enddoc:
+
+# Used to keep track of file offsets in IO#sendfile_nonblock + evented
+# models.  We always maintain our own file offsets in userspace because
+# because sendfile() implementations offer pread()-like idempotency for
+# concurrency (multiple clients can read the same underlying file handle).
+class Rainbows::StreamFile < Struct.new(:offset, :to_io)
+
+  def close
+    to_io.close
+    self.to_io = nil
+  end
+end

data/lib/rainbows/tee_input.rb

@@ -1,4 +1,5 @@
 # -*- encoding: binary -*-
+# :enddoc:
 module Rainbows
 
   # acts like tee(1) on an input input to provide a input-like stream

data/lib/rainbows/thread_pool.rb

@@ -25,7 +25,7 @@ module Rainbows
 
     include Base
 
-    def worker_loop(worker)
+    def worker_loop(worker) # :nodoc:
       init_worker_process(worker)
       pool = (1..worker_connections).map do
         Thread.new { LISTENERS.size == 1 ? sync_worker : async_worker }
@@ -41,8 +41,8 @@ module Rainbows
       join_threads(pool)
     end
 
-    def sync_worker
-      s = LISTENERS.first
+    def sync_worker # :nodoc:
+      s = LISTENERS[0]
       begin
         c = Rainbows.sync_accept(s) and process_client(c)
       rescue => e
@@ -50,14 +50,14 @@ module Rainbows
       end while G.alive
     end
 
-    def async_worker
+    def async_worker # :nodoc:
       begin
         # TODO: check if select() or accept() is a problem on large
         # SMP systems under Ruby 1.9.  Hundreds of native threads
         # all working off the same socket could be a thundering herd
         # problem.  On the other hand, a thundering herd may not
         # even incur as much overhead as an extra Mutex#synchronize
-        ret = IO.select(LISTENERS, nil, nil, 1) and ret.first.each do |s|
+        ret = IO.select(LISTENERS, nil, nil, 1) and ret[0].each do |s|
           s = Rainbows.accept(s) and process_client(s)
         end
       rescue Errno::EINTR
@@ -66,11 +66,16 @@ module Rainbows
       end while G.alive
     end
 
-    def join_threads(threads)
+    def join_threads(threads) # :nodoc:
       G.quit!
       threads.delete_if do |thr|
         G.tick
-        thr.alive? ? thr.join(0.01) : true
+        begin
+          thr.run
+          thr.join(0.01)
+        rescue
+          true
+        end
       end until threads.empty?
     end
 

data/lib/rainbows/thread_spawn.rb

@@ -17,10 +17,9 @@ module Rainbows
   # capabilities
 
   module ThreadSpawn
-
     include Base
 
-    def accept_loop(klass)
+    def accept_loop(klass) #:nodoc:
       lock = Mutex.new
       limit = worker_connections
       LISTENERS.each do |l|
@@ -55,7 +54,7 @@ module Rainbows
       sleep 1 while G.tick || lock.synchronize { G.cur > 0 }
     end
 
-    def worker_loop(worker)
+    def worker_loop(worker) #:nodoc:
       init_worker_process(worker)
       accept_loop(Thread)
     end

data/lib/rainbows/writer_thread_pool.rb

@@ -24,7 +24,7 @@ module Rainbows
 
     # used to wrap a BasicSocket to use with +q+ for all writes
     # this is compatible with IO.select
-    class QueueSocket < Struct.new(:to_io, :q)
+    class QueueSocket < Struct.new(:to_io, :q) # :nodoc:
       def readpartial(size, buf = "")
         to_io.readpartial(size, buf)
       end
@@ -46,19 +46,25 @@ module Rainbows
       end
     end
 
-    def write_body(qclient, body)
-      qclient.q << [ qclient.to_io, :body, body ]
+    module Response # :nodoc:
+      def write_body(qclient, body)
+        qclient.q << [ qclient.to_io, :body, body ]
+      end
     end
 
     @@nr = 0
     @@q = nil
 
-    def process_client(client)
+    def process_client(client) # :nodoc:
       @@nr += 1
       super(QueueSocket[client, @@q[@@nr %= @@q.size]])
     end
 
-    def worker_loop(worker)
+    def worker_loop(worker) # :nodoc:
+      Rainbows::Response.setup(self.class)
+      self.class.__send__(:alias_method, :sync_write_body, :write_body)
+      self.class.__send__(:include, Response)
+
       # we have multiple, single-thread queues since we don't want to
       # interleave writes from the same client
       qp = (1..worker_connections).map do |n|
@@ -66,13 +72,13 @@ module Rainbows
           begin
             io, arg1, arg2 = response
             case arg1
-            when :body then Base.write_body(io, arg2)
+            when :body then sync_write_body(io, arg2)
             when :close then io.close unless io.closed?
             else
               io.write(arg1)
             end
           rescue => err
-            Error.app(err)
+            Error.write(io, err)
           end
         end
       end

data/lib/rainbows/writer_thread_spawn.rb

@@ -23,11 +23,13 @@ module Rainbows
   module WriterThreadSpawn
     include Base
 
-    CUR = {}
+    CUR = {} # :nodoc:
 
     # used to wrap a BasicSocket to use with +q+ for all writes
     # this is compatible with IO.select
-    class MySocket < Struct.new(:to_io, :q, :thr)
+    class MySocket < Struct.new(:to_io, :q, :thr)  # :nodoc: all
+      include Rainbows::Response
+
       def readpartial(size, buf = "")
         to_io.readpartial(size, buf)
       end
@@ -51,7 +53,7 @@ module Rainbows
             begin
               arg1, arg2 = response
               case arg1
-              when :body then Base.write_body(io, arg2)
+              when :body then write_body(io, arg2)
               when :close
                 io.close unless io.closed?
                 break
@@ -59,7 +61,7 @@ module Rainbows
                 io.write(arg1)
               end
             rescue => e
-              Error.app(e)
+              Error.write(io, e)
             end
           end
           CUR.delete(Thread.current)
@@ -71,7 +73,7 @@ module Rainbows
         (self.q ||= queue_writer) << buf
       end
 
-      def write_body(body)
+      def queue_body(body)
         (self.q ||= queue_writer) << [ :body, body ]
       end
 
@@ -88,16 +90,17 @@ module Rainbows
       end
     end
 
-    def write_body(my_sock, body)
-      my_sock.write_body(body)
+    def write_body(my_sock, body) # :nodoc:
+      my_sock.queue_body(body)
     end
 
-    def process_client(client)
+    def process_client(client) # :nodoc:
       super(MySocket[client])
     end
 
-    def worker_loop(worker)
+    def worker_loop(worker)  # :nodoc:
       MySocket.const_set(:MAX, worker_connections)
+      Rainbows::Response.setup(MySocket)
       super(worker) # accept loop from Unicorn
       CUR.delete_if do |t,q|
         q << nil