rainbows 0.97.0 → 1.0.0pre1

data/lib/rainbows/fiber/rev.rb

@@ -4,168 +4,10 @@ require 'rev'
 require 'rainbows/fiber'
 require 'rainbows/fiber/io'
 
-module Rainbows::Fiber
-  module Rev
-    G = Rainbows::G
-
-    # keep-alive timeout class
-    class Kato < ::Rev::TimerWatcher
-      def initialize
-        @watch = []
-        super(1, true)
-      end
-
-      def <<(fiber)
-        @watch << fiber
-        enable unless enabled?
-      end
-
-      def on_timer
-        @watch.uniq!
-        while f = @watch.shift
-          f.resume if f.alive?
-        end
-        disable
-      end
-    end
-
-    class Heartbeat < ::Rev::TimerWatcher
-      def on_timer
-        exit if (! G.tick && G.cur <= 0)
-      end
-    end
-
-    class Sleeper < ::Rev::TimerWatcher
-
-      def initialize(seconds)
-        @f = ::Fiber.current
-        super(seconds, false)
-        attach(::Rev::Loop.default)
-        ::Fiber.yield
-      end
-
-      def on_timer
-        @f.resume
-      end
-    end
-
-    class Server < ::Rev::IOWatcher
-      include Unicorn
-      include Rainbows
-      include Rainbows::Const
-      include Rainbows::Response
-      include Rainbows::Acceptor
-      FIO = Rainbows::Fiber::IO
-
-      def to_io
-        @io
-      end
-
-      def initialize(io)
-        @io = io
-        super(self, :r)
-      end
-
-      def close
-        detach if attached?
-        @io.close
-      end
-
-      def on_readable
-        return if G.cur >= MAX
-        c = accept(@io) and ::Fiber.new { process(c) }.resume
-      end
-
-      def process(io)
-        G.cur += 1
-        client = FIO.new(io, ::Fiber.current)
-        buf = client.read_timeout or return
-        hp = HttpParser.new
-        env = {}
-        remote_addr = Rainbows.addr(io)
-
-        begin # loop
-          buf << (client.read_timeout or return) until hp.headers(env, buf)
-
-          env[CLIENT_IO] = client
-          env[RACK_INPUT] = 0 == hp.content_length ?
-                    HttpRequest::NULL_IO : TeeInput.new(client, env, hp, buf)
-          env[REMOTE_ADDR] = remote_addr
-          status, headers, body = APP.call(env.update(RACK_DEFAULTS))
-
-          if 100 == status.to_i
-            client.write(EXPECT_100_RESPONSE)
-            env.delete(HTTP_EXPECT)
-            status, headers, body = APP.call(env)
-          end
-
-          if hp.headers?
-            headers = HH.new(headers)
-            range = make_range!(env, status, headers) and status = range.shift
-            headers[CONNECTION] = if hp.keepalive? && G.alive
-              KEEP_ALIVE
-            else
-              env = false
-              CLOSE
-            end
-            client.write(response_header(status, headers))
-          end
-          write_body(client, body, range)
-        end while env && env.clear && hp.reset.nil?
-      rescue => e
-        Error.write(io, e)
-      ensure
-        G.cur -= 1
-        client.close
-      end
-    end
-  end
-
-  class IO # see rainbows/fiber/io for original definition
-
-    class Watcher < ::Rev::IOWatcher
-      def initialize(fio, flag)
-        @fiber = fio.f
-        super(fio, flag)
-        attach(::Rev::Loop.default)
-      end
-
-      def on_readable
-        @fiber.resume
-      end
-
-      alias on_writable on_readable
-    end
-
-    undef_method :wait_readable
-    undef_method :wait_writable
-    undef_method :close
-
-    def initialize(*args)
-      super
-      @r = @w = false
-    end
-
-    def close
-      @w.detach if @w
-      @r.detach if @r
-      @r = @w = false
-      to_io.close unless to_io.closed?
-    end
-
-    def wait_writable
-      @w ||= Watcher.new(self, :w)
-      @w.enable unless @w.enabled?
-      ::Fiber.yield
-      @w.disable
-    end
-
-    def wait_readable
-      @r ||= Watcher.new(self, :r)
-      @r.enable unless @r.enabled?
-      KATO << f
-      ::Fiber.yield
-      @r.disable
-    end
-  end
+module Rainbows::Fiber::Rev
+  autoload :Heartbeat, 'rainbows/fiber/rev/heartbeat'
+  autoload :Kato, 'rainbows/fiber/rev/kato'
+  autoload :Server, 'rainbows/fiber/rev/server'
+  autoload :Sleeper, 'rainbows/fiber/rev/sleeper'
 end
+require 'rainbows/fiber/rev/methods'

data/lib/rainbows/fiber.rb

@@ -6,11 +6,29 @@ rescue LoadError
   defined?(NeverBlock) or raise
 end
 
-module Rainbows
+# core module for all things that use Fibers in Rainbows!
+module Rainbows::Fiber
 
-  # core module for all things that use Fibers in Rainbows!
-  module Fiber
-    autoload :Base, 'rainbows/fiber/base'
-    autoload :Queue, 'rainbows/fiber/queue'
+  # blocked readers (key: fileno, value: Rainbows::Fiber::IO object)
+  RD = []
+
+  # blocked writers (key: fileno, value: Rainbows::Fiber::IO object)
+  WR = []
+
+  # sleeping fibers go here (key: Fiber object, value: wakeup time)
+  ZZ = {}
+
+  # puts the current Fiber into uninterruptible sleep for at least
+  # +seconds+.  Unlike Kernel#sleep, this it is not possible to sleep
+  # indefinitely to be woken up (nobody wants that in a web server,
+  # right?).  Calling this directly is deprecated, use
+  # Rainbows.sleep(seconds) instead.
+  def self.sleep(seconds)
+    ZZ[::Fiber.current] = Time.now + seconds
+    ::Fiber.yield
   end
+
+  autoload :Base, 'rainbows/fiber/base'
+  autoload :Queue, 'rainbows/fiber/queue'
+  autoload :IO, 'rainbows/fiber/io'
 end

data/lib/rainbows/fiber_pool.rb

@@ -1,45 +1,39 @@
 # -*- encoding: binary -*-
 require 'rainbows/fiber'
 
-module Rainbows
+# A Fiber-based concurrency model for Ruby 1.9.  This uses a pool of
+# Fibers to handle client IO to run the application and the root Fiber
+# for scheduling and connection acceptance.  The pool size is equal to
+# the number of +worker_connections+.  Compared to the ThreadPool
+# model, Fibers are very cheap in terms of memory usage so you can
+# have more active connections.  This model supports a streaming
+# "rack.input" with lightweight concurrency.  Applications are
+# strongly advised to wrap all slow IO objects (sockets, pipes) using
+# the Rainbows::Fiber::IO class whenever possible.
+module Rainbows::FiberPool
+  include Rainbows::Fiber::Base
 
-  # A Fiber-based concurrency model for Ruby 1.9.  This uses a pool of
-  # Fibers to handle client IO to run the application and the root Fiber
-  # for scheduling and connection acceptance.  The pool size is equal to
-  # the number of +worker_connections+.  Compared to the ThreadPool
-  # model, Fibers are very cheap in terms of memory usage so you can
-  # have more active connections.  This model supports a streaming
-  # "rack.input" with lightweight concurrency.  Applications are
-  # strongly advised to wrap all slow IO objects (sockets, pipes) using
-  # the Rainbows::Fiber::IO class whenever possible.
+  def worker_loop(worker) # :nodoc:
+    init_worker_process(worker)
+    pool = []
+    worker_connections.times {
+      Fiber.new {
+        process(Fiber.yield) while pool << Fiber.current
+      }.resume # resume to hit ::Fiber.yield so it waits on a client
+    }
+    Rainbows::Fiber::Base.setup(self.class, app)
 
-  module FiberPool
-    include Fiber::Base
-    include Rainbows::Acceptor
-
-    def worker_loop(worker) # :nodoc:
-      init_worker_process(worker)
-      pool = []
-      worker_connections.times {
-        ::Fiber.new {
-          process_client(::Fiber.yield) while pool << ::Fiber.current
-        }.resume # resume to hit ::Fiber.yield so it waits on a client
-      }
-      Fiber::Base.setup(self.class, app)
-
-      begin
-        schedule do |l|
-          fib = pool.shift or break # let another worker process take it
-          if io = accept(l)
-            fib.resume(Fiber::IO.new(io, fib))
-          else
-            pool << fib
-          end
+    begin
+      schedule do |l|
+        fib = pool.shift or break # let another worker process take it
+        if io = l.kgio_tryaccept
+          fib.resume(io)
+        else
+          pool << fib
         end
-      rescue => e
-        Error.listen_loop(e)
-      end while G.alive || G.cur > 0
-    end
-
+      end
+    rescue => e
+      Rainbows::Error.listen_loop(e)
+    end while G.alive || G.cur > 0
   end
 end

data/lib/rainbows/fiber_spawn.rb

@@ -1,35 +1,28 @@
 # -*- encoding: binary -*-
 require 'rainbows/fiber'
 
-module Rainbows
+# Simple Fiber-based concurrency model for 1.9.  This spawns a new
+# Fiber for every incoming client connection and the root Fiber for
+# scheduling and connection acceptance.  This exports a streaming
+# "rack.input" with lightweight concurrency.  Applications are
+# strongly advised to wrap all slow IO objects (sockets, pipes) using
+# the Rainbows::Fiber::IO class whenever possible.
+module Rainbows::FiberSpawn
+  include Rainbows::Fiber::Base
 
-  # Simple Fiber-based concurrency model for 1.9.  This spawns a new
-  # Fiber for every incoming client connection and the root Fiber for
-  # scheduling and connection acceptance.  This exports a streaming
-  # "rack.input" with lightweight concurrency.  Applications are
-  # strongly advised to wrap all slow IO objects (sockets, pipes) using
-  # the Rainbows::Fiber::IO class whenever possible.
-
-  module FiberSpawn
-    include Fiber::Base
-    include Rainbows::Acceptor
-
-    def worker_loop(worker) # :nodoc:
-      init_worker_process(worker)
-      Fiber::Base.setup(self.class, app)
-      limit = worker_connections
-      fio = Rainbows::Fiber::IO
-
-      begin
-        schedule do |l|
-          break if G.cur >= limit
-          io = accept(l) or next
-          ::Fiber.new { process_client(fio.new(io, ::Fiber.current)) }.resume
-        end
-      rescue => e
-        Error.listen_loop(e)
-      end while G.alive || G.cur > 0
-    end
+  def worker_loop(worker) # :nodoc:
+    init_worker_process(worker)
+    Rainbows::Fiber::Base.setup(self.class, app)
+    limit = worker_connections
 
+    begin
+      schedule do |l|
+        break if G.cur >= limit
+        io = l.kgio_tryaccept or next
+        Fiber.new { process(io) }.resume
+      end
+    rescue => e
+      Rainbows::Error.listen_loop(e)
+    end while G.alive || G.cur > 0
   end
 end

data/lib/rainbows/http_server.rb

@@ -1,97 +1,97 @@
 # -*- encoding: binary -*-
 # :enddoc:
-module Rainbows
 
-  class HttpServer < ::Unicorn::HttpServer
-    include Rainbows
+class Rainbows::HttpServer < Unicorn::HttpServer
+  G = Rainbows::G
 
-    class << self
-      def setup(block)
-        G.server.instance_eval(&block)
-      end
-    end
-
-    def initialize(app, options)
-      G.server = self
-      rv = super(app, options)
-      defined?(@use) or use(:Base)
-      @worker_connections ||= MODEL_WORKER_CONNECTIONS[@use]
-    end
+  def self.setup(block)
+    G.server.instance_eval(&block)
+  end
 
-    def reopen_worker_logs(worker_nr)
-      logger.info "worker=#{worker_nr} reopening logs..."
-      Unicorn::Util.reopen_logs
-      logger.info "worker=#{worker_nr} done reopening logs"
-      rescue
-        G.quit! # let the master reopen and refork us
-    end
+  def initialize(app, options)
+    G.server = self
+    @logger = Unicorn::Configurator::DEFAULTS[:logger]
+    rv = super(app, options)
+    defined?(@use) or use(:Base)
+    @worker_connections ||= Rainbows::MODEL_WORKER_CONNECTIONS[@use]
+  end
 
-    #:stopdoc:
-    #
-    # Add one second to the timeout since our fchmod heartbeat is less
-    # precise (and must be more conservative) than Unicorn does.  We
-    # handle many clients per process and can't chmod on every
-    # connection we accept without wasting cycles.  That added to the
-    # fact that we let clients keep idle connections open for long
-    # periods of time means we have to chmod at a fixed interval.
-    def timeout=(nr)
-      super(nr + 1)
-    end
-    #:startdoc:
+  def reopen_worker_logs(worker_nr)
+    logger.info "worker=#{worker_nr} reopening logs..."
+    Unicorn::Util.reopen_logs
+    logger.info "worker=#{worker_nr} done reopening logs"
+    rescue
+      G.quit! # let the master reopen and refork us
+  end
 
-    def use(*args)
-      model = args.shift or return @use
-      mod = begin
-        Rainbows.const_get(model)
-      rescue NameError
-        raise ArgumentError, "concurrency model #{model.inspect} not supported"
-      end
+  #:stopdoc:
+  #
+  # Add one second to the timeout since our fchmod heartbeat is less
+  # precise (and must be more conservative) than Unicorn does.  We
+  # handle many clients per process and can't chmod on every
+  # connection we accept without wasting cycles.  That added to the
+  # fact that we let clients keep idle connections open for long
+  # periods of time means we have to chmod at a fixed interval.
+  def timeout=(nr)
+    @timeout = nr + 1
+  end
+  #:startdoc:
 
-      Module === mod or
-        raise ArgumentError, "concurrency model #{model.inspect} not supported"
-      extend(mod)
-      args.each do |opt|
-        case opt
-        when Hash; O.update(opt)
-        when Symbol; O[opt] = true
-        else; raise ArgumentError, "can't handle option: #{opt.inspect}"
-        end
-      end
-      mod.setup if mod.respond_to?(:setup)
-      Const::RACK_DEFAULTS['rainbows.model'] = @use = model.to_sym
-      Const::RACK_DEFAULTS['rack.multithread'] = !!(model.to_s =~ /Thread/)
+  def use(*args)
+    model = args.shift or return @use
+    mod = begin
+      Rainbows.const_get(model)
+    rescue NameError => e
+      logger.error "error loading #{model.inspect}: #{e}"
+      e.backtrace.each { |l| logger.error l }
+      raise ArgumentError, "concurrency model #{model.inspect} not supported"
+    end
 
-      case @use
-      when :Rev, :EventMachine, :NeverBlock
-        Const::RACK_DEFAULTS['rainbows.autochunk'] = true
+    Module === mod or
+      raise ArgumentError, "concurrency model #{model.inspect} not supported"
+    extend(mod)
+    args.each do |opt|
+      case opt
+      when Hash; O.update(opt)
+      when Symbol; O[opt] = true
+      else; raise ArgumentError, "can't handle option: #{opt.inspect}"
       end
     end
-
-    def worker_connections(*args)
-      return @worker_connections if args.empty?
-      nr = args[0]
-      (Integer === nr && nr > 0) or
-        raise ArgumentError, "worker_connections must be a positive Integer"
-      @worker_connections = nr
+    mod.setup if mod.respond_to?(:setup)
+    new_defaults = {
+      'rainbows.model' => (@use = model.to_sym),
+      'rack.multithread' => !!(model.to_s =~ /Thread/),
+    }
+    case @use
+    when :Rev, :EventMachine, :NeverBlock
+      new_defaults['rainbows.autochunk'] = true
     end
+    Rainbows::Const::RACK_DEFAULTS.update(new_defaults)
+  end
 
-    def keepalive_timeout(nr)
-      (Integer === nr && nr >= 0) or
-        raise ArgumentError, "keepalive must be a non-negative Integer"
-      G.kato = nr
-    end
+  def worker_connections(*args)
+    return @worker_connections if args.empty?
+    nr = args[0]
+    (Integer === nr && nr > 0) or
+      raise ArgumentError, "worker_connections must be a positive Integer"
+    @worker_connections = nr
+  end
 
-    def client_max_body_size(nr)
-      err = "client_max_body_size must be nil or a non-negative Integer"
-      case nr
-      when nil
-      when Integer
-        nr >= 0 or raise ArgumentError, err
-      else
-        raise ArgumentError, err
-      end
-      Rainbows.max_bytes = nr
-    end
+  def keepalive_timeout(nr)
+    (Integer === nr && nr >= 0) or
+      raise ArgumentError, "keepalive must be a non-negative Integer"
+    G.kato = nr
   end
 
+  def client_max_body_size(nr)
+    err = "client_max_body_size must be nil or a non-negative Integer"
+    case nr
+    when nil
+    when Integer
+      nr >= 0 or raise ArgumentError, err
+    else
+      raise ArgumentError, err
+    end
+    Rainbows.max_bytes = nr
+  end
 end

data/lib/rainbows/max_body.rb

@@ -1,42 +1,42 @@
 # -*- encoding: binary -*-
 # :enddoc:
-module Rainbows
 
 # middleware used to enforce client_max_body_size for TeeInput users,
 # there is no need to configure this middleware manually, it will
 # automatically be configured for you based on the client_max_body_size
 # setting
-class MaxBody < Struct.new(:app)
+class Rainbows::MaxBody < Struct.new(:app)
 
   # this is meant to be included in Rainbows::TeeInput (and derived
   # classes) to limit body sizes
   module Limit
-    Util = Unicorn::Util
+    TmpIO = Unicorn::TmpIO
+    MAX_BODY = Rainbows::Const::MAX_BODY
 
-    def initialize(socket, req, parser, buf)
-      self.len = parser.content_length
+    def initialize(socket, request)
+      @parser = request
+      @buf = request.buf
+      @env = request.env
+      @len = request.content_length
 
       max = Rainbows.max_bytes # never nil, see MaxBody.setup
-      if len && len > max
-        socket.write(Const::ERROR_413_RESPONSE)
+      if @len && @len > max
+        socket.write(Rainbows::Const::ERROR_413_RESPONSE)
         socket.close
-        raise IOError, "Content-Length too big: #{len} > #{max}", []
+        raise IOError, "Content-Length too big: #@len > #{max}", []
       end
 
-      self.req = req
-      self.parser = parser
-      self.buf = buf
-      self.socket = socket
-      self.buf2 = ""
-      if buf.size > 0
-        parser.filter_body(buf2, buf) and finalize_input
-        buf2.size > max and raise IOError, "chunked request body too big", []
+      @socket = socket
+      @buf2 = ""
+      if @buf.size > 0
+        parser.filter_body(@buf2, @buf) and finalize_input
+        @buf2.size > max and raise IOError, "chunked request body too big", []
       end
-      self.tmp = len && len < Const::MAX_BODY ? StringIO.new("") : Util.tmpio
-      if buf2.size > 0
-        tmp.write(buf2)
-        tmp.seek(0)
-        max -= buf2.size
+      @tmp = @len && @len < MAX_BODY ? StringIO.new("") : TmpIO.new
+      if @buf2.size > 0
+        @tmp.write(@buf2)
+        @tmp.rewind
+        max -= @buf2.size
       end
       @max_body = max
     end
@@ -46,7 +46,7 @@ class MaxBody < Struct.new(:app)
       if rv && ((@max_body -= rv.size) < 0)
         # make HttpParser#keepalive? => false to force an immediate disconnect
         # after we write
-        parser.reset
+        @parser.reset
         throw :rainbows_EFBIG
       end
       rv
@@ -58,15 +58,15 @@ class MaxBody < Struct.new(:app)
   # if it's reconfigured
   def self.setup
     Rainbows.max_bytes or return
-    case G.server.use
+    case Rainbows::G.server.use
     when :Rev, :EventMachine, :NeverBlock
       return
     end
 
-    TeeInput.class_eval { include Limit }
+    Rainbows::TeeInput.__send__(:include, Limit)
 
     # force ourselves to the outermost middleware layer
-    G.server.app = MaxBody.new(G.server.app)
+    Rainbows::G.server.app = self.new(Rainbows::G.server.app)
   end
 
   # Rack response returned when there's an error
@@ -78,6 +78,4 @@ class MaxBody < Struct.new(:app)
   def call(env)
     catch(:rainbows_EFBIG) { app.call(env) } || err(env)
   end
-
-end # class
-end # module
+end

data/lib/rainbows/process_client.rb

@@ -0,0 +1,61 @@
+# -*- encoding: binary -*-
+# :enddoc:
+module Rainbows::ProcessClient
+  G = Rainbows::G
+  include Rainbows::Response
+  HttpParser = Unicorn::HttpParser
+  NULL_IO = Unicorn::HttpRequest::NULL_IO
+  RACK_INPUT = Unicorn::HttpRequest::RACK_INPUT
+  TeeInput = Rainbows::TeeInput
+  include Rainbows::Const
+
+  def wait_headers_readable(client)
+    IO.select([client], nil, nil, G.kato)
+  end
+
+  # once a client is accepted, it is processed in its entirety here
+  # in 3 easy steps: read request, call app, write app response
+  # this is used by synchronous concurrency models
+  #   Base, ThreadSpawn, ThreadPool
+  def process_client(client) # :nodoc:
+    hp = HttpParser.new
+    client.kgio_read!(16384, buf = hp.buf)
+    remote_addr = client.kgio_addr
+
+    begin # loop
+      until env = hp.parse
+        wait_headers_readable(client) or return
+        buf << client.kgio_read!(16384)
+      end
+
+      env[CLIENT_IO] = client
+      env[RACK_INPUT] = 0 == hp.content_length ?
+                        NULL_IO : TeeInput.new(client, hp)
+      env[REMOTE_ADDR] = remote_addr
+      status, headers, body = APP.call(env.update(RACK_DEFAULTS))
+
+      if 100 == status.to_i
+        client.write(EXPECT_100_RESPONSE)
+        env.delete(HTTP_EXPECT)
+        status, headers, body = APP.call(env)
+      end
+
+      if hp.headers?
+        headers = HH.new(headers)
+        range = make_range!(env, status, headers) and status = range.shift
+        env = hp.keepalive? && G.alive
+        headers[CONNECTION] = env ? KEEP_ALIVE : CLOSE
+        client.write(response_header(status, headers))
+      end
+      write_body(client, body, range)
+    end while env && hp.reset.nil?
+  # if we get any error, try to write something back to the client
+  # assuming we haven't closed the socket, but don't get hung up
+  # if the socket is already closed or broken.  We'll always ensure
+  # the socket is closed at the end of this function
+  rescue => e
+    Rainbows::Error.write(client, e)
+  ensure
+    client.close unless client.closed?
+  end
+end