rainbows 3.2.0 → 3.3.0

data/lib/rainbows/epoll/client.rb

@@ -10,7 +10,7 @@ module Rainbows::Epoll::Client
   OUT = SleepyPenguin::Epoll::OUT | SleepyPenguin::Epoll::ET
   KATO = {}
   KATO.compare_by_identity if KATO.respond_to?(:compare_by_identity)
-  KEEPALIVE_TIMEOUT = Rainbows.keepalive_timeout
+  Rainbows.config!(self, :keepalive_timeout)
   EP = Rainbows::Epoll::EP
   @@last_expire = Time.now
 
@@ -33,7 +33,7 @@ module Rainbows::Epoll::Client
   end
 
   def on_readable
-    case rv = kgio_tryread(16384, RBUF)
+    case rv = kgio_tryread(CLIENT_HEADER_BUFFER_SIZE, RBUF)
     when String
       on_read(rv)
       return if @wr_queue[0] || closed?
@@ -226,7 +226,7 @@ module Rainbows::Epoll::Client
     else # nil => EOF
       return pipe.close # nil
     end while true
-    rescue => e
+    rescue
       pipe.close
       raise
   end

data/lib/rainbows/ev_core.rb

@@ -9,6 +9,7 @@ module Rainbows::EvCore
   autoload :CapInput, 'rainbows/ev_core/cap_input'
   RBUF = ""
   Z = "".freeze
+  Rainbows.config!(self, :client_header_buffer_size)
 
   # Apps may return this Rack response: AsyncResponse = [ -1, {}, [] ]
   ASYNC_CALLBACK = "async.callback".freeze
@@ -132,7 +133,7 @@ module Rainbows::EvCore
   end
 
   def mkinput
-    max = Rainbows.max_bytes
+    max = Rainbows.server.client_max_body_size
     len = @hp.content_length
     if len
       if max && (len > max)

data/lib/rainbows/event_machine.rb

@@ -40,6 +40,10 @@ EM::VERSION >= '0.12.10' or abort 'eventmachine 0.12.10 is required'
 # the Rack application to process data as it arrives.  This means
 # "rack.input" will be fully buffered in memory or to a temporary file
 # before the application is entered.
+#
+# === RubyGem Requirements
+#
+# * event_machine 0.12.10
 module Rainbows::EventMachine
   autoload :ResponsePipe, 'rainbows/event_machine/response_pipe'
   autoload :ResponseChunkPipe, 'rainbows/event_machine/response_chunk_pipe'

data/lib/rainbows/event_machine/client.rb

@@ -2,6 +2,7 @@
 # :enddoc:
 class Rainbows::EventMachine::Client < EM::Connection
   include Rainbows::EvCore
+  Rainbows.config!(self, :keepalive_timeout)
 
   def initialize(io)
     @_io = io
@@ -87,7 +88,7 @@ class Rainbows::EventMachine::Client < EM::Connection
     if alive
       if @deferred.nil?
         if @buf.empty?
-          set_comm_inactivity_timeout(Rainbows.keepalive_timeout)
+          set_comm_inactivity_timeout(KEEPALIVE_TIMEOUT)
         else
           EM.next_tick { receive_data(nil) }
         end

data/lib/rainbows/fiber.rb

@@ -8,6 +8,11 @@ end
 # :startdoc:
 
 # core namespace for all things that use Fibers in \Rainbows!
+#
+# It's generally not recommended to use any of this in your applications
+# unless you're willing to accept breakage.  Most of this is very
+# difficult-to-use, fragile and we don't have much time to devote to
+# supporting these in the future.
 module Rainbows::Fiber
 
   # :stopdoc:

data/lib/rainbows/fiber/base.rb

@@ -64,6 +64,7 @@ module Rainbows::Fiber::Base
   end
 
   def self.setup(klass, app)
+    Rainbows::Client.__send__(:include, Rainbows::Fiber::IO::Methods)
     require 'rainbows/fiber/body'
     Rainbows::Client.__send__(:include, Rainbows::Fiber::Body)
     self.const_set(:APP, app)

data/lib/rainbows/fiber/coolio/methods.rb

@@ -38,7 +38,6 @@ end
 
 [
   Rainbows::Fiber::IO,
-  Rainbows::Client,
   # the next two trigger autoload, ugh, oh well...
   Rainbows::Fiber::IO::Socket,
   Rainbows::Fiber::IO::Pipe

data/lib/rainbows/fiber/io.rb

@@ -1,13 +1,18 @@
 # -*- encoding: binary -*-
 
-# A Fiber-aware IO class, gives users the illusion of a synchronous
-# interface that yields away from the current Fiber whenever
+# A \Fiber-aware IO class, gives users the illusion of a synchronous
+# interface that yields away from the current \Fiber whenever
 # the underlying descriptor is blocked on reads or write
 #
+# It's not recommended to use any of this in your applications
+# unless you're willing to accept breakage.  Most of this is very
+# difficult-to-use, fragile and we don't have much time to devote to
+# supporting these in the future.
+#
 # This is a stable, legacy interface and should be preserved for all
 # future versions of Rainbows!  However, new apps should use
-# Rainbows::Fiber::IO::Socket or Rainbows::Fiber::IO::Pipe instead.
-
+# Rainbows::Fiber::IO::Socket or Rainbows::Fiber::IO::Pipe instead
+# (or better yet, avoid any of the Rainbows::Fiber* stuff).
 class Rainbows::Fiber::IO
   attr_accessor :to_io
 
@@ -45,7 +50,7 @@ class Rainbows::Fiber::IO
   end
 
   def write(buf)
-    case rv = Kgio.trywrite(buf)
+    case rv = Kgio.trywrite(@to_io, buf)
     when String
       buf = rv
     when :wait_writable
@@ -96,7 +101,6 @@ end
 # :stopdoc:
 require 'rainbows/fiber/io/methods'
 require 'rainbows/fiber/io/compat'
-Rainbows::Client.__send__(:include, Rainbows::Fiber::IO::Methods)
 class Rainbows::Fiber::IO
   include Rainbows::Fiber::IO::Compat
   include Rainbows::Fiber::IO::Methods

data/lib/rainbows/fiber/io/pipe.rb

@@ -1,7 +1,12 @@
 # -*- encoding: binary -*-
 # A Fiber-aware Pipe class, gives users the illusion of a synchronous
 # interface that yields away from the current Fiber whenever
-# the underlying descriptor is blocked on reads or write
+# the underlying descriptor is blocked on reads or write.
+#
+# It's not recommended to use any of this in your applications
+# unless you're willing to accept breakage.  Most of this is very
+# difficult-to-use, fragile and we don't have much time to devote to
+# supporting these in the future.
 class Rainbows::Fiber::IO::Pipe < Kgio::Pipe
   include Rainbows::Fiber::IO::Methods
 end

data/lib/rainbows/fiber/io/socket.rb

@@ -1,7 +1,12 @@
 # -*- encoding: binary -*-
 # A Fiber-aware Socket class, gives users the illusion of a synchronous
 # interface that yields away from the current Fiber whenever
-# the underlying descriptor is blocked on reads or write
+# the underlying descriptor is blocked on reads or write.
+#
+# It's not recommended to use any of this in your applications
+# unless you're willing to accept breakage.  Most of this is very
+# difficult-to-use, fragile and we don't have much time to devote to
+# supporting these in the future.
 class Rainbows::Fiber::IO::Socket < Kgio::Socket
   include Rainbows::Fiber::IO::Methods
 end

data/lib/rainbows/fiber_pool.rb

@@ -3,13 +3,18 @@ require 'rainbows/fiber'
 
 # A Fiber-based concurrency model for Ruby 1.9.  This uses a pool of
 # Fibers to handle client IO to run the application and the root Fiber
-# for scheduling and connection acceptance.  The pool size is equal to
-# the number of +worker_connections+.  Compared to the ThreadPool
-# model, Fibers are very cheap in terms of memory usage so you can
-# have more active connections.  This model supports a streaming
-# "rack.input" with lightweight concurrency.  Applications are
-# strongly advised to wrap all slow IO objects (sockets, pipes) using
-# the Rainbows::Fiber::IO class whenever possible.
+# for scheduling and connection acceptance.
+#
+# This concurrency model is difficult to use with existing applications,
+# lacks third-party support, and is thus NOT recommended.
+#
+# The pool size is equal to the number of +worker_connections+.
+# Compared to the ThreadPool model, Fibers are very cheap in terms of
+# memory usage so you can have more active connections.  This model
+# supports a streaming "rack.input" with lightweight concurrency.
+# Applications are strongly advised to wrap all slow IO objects
+# (sockets, pipes) using the Rainbows::Fiber::IO class whenever
+# possible.
 module Rainbows::FiberPool
   include Rainbows::Fiber::Base
 

data/lib/rainbows/fiber_spawn.rb

@@ -1,12 +1,17 @@
 # -*- encoding: binary -*-
 require 'rainbows/fiber'
 
-# Simple Fiber-based concurrency model for 1.9.  This spawns a new
-# Fiber for every incoming client connection and the root Fiber for
-# scheduling and connection acceptance.  This exports a streaming
-# "rack.input" with lightweight concurrency.  Applications are
-# strongly advised to wrap all slow IO objects (sockets, pipes) using
-# the Rainbows::Fiber::IO class whenever possible.
+# Simple Fiber-based concurrency model for 1.9.  This spawns a new Fiber
+# for every incoming client connection and the root Fiber for scheduling
+# and connection acceptance.
+#
+# This concurrency model is difficult to use with existing applications,
+# lacks third-party support, and is thus NOT recommended.
+#
+# This exports a streaming "rack.input" with lightweight concurrency.
+# Applications are strongly advised to wrap all slow IO objects
+# (sockets, pipes) using the Rainbows::Fiber::IO class whenever
+# possible.
 module Rainbows::FiberSpawn
   include Rainbows::Fiber::Base
 

data/lib/rainbows/http_server.rb

@@ -2,6 +2,13 @@
 # :enddoc:
 
 class Rainbows::HttpServer < Unicorn::HttpServer
+  attr_accessor :copy_stream
+  attr_accessor :worker_connections
+  attr_accessor :keepalive_timeout
+  attr_accessor :client_header_buffer_size
+  attr_accessor :client_max_body_size
+  attr_reader :use
+
   def self.setup(block)
     Rainbows.server.instance_eval(&block)
   end
@@ -9,9 +16,9 @@ class Rainbows::HttpServer < Unicorn::HttpServer
   def initialize(app, options)
     Rainbows.server = self
     @logger = Unicorn::Configurator::DEFAULTS[:logger]
-    rv = super(app, options)
-    defined?(@use) or use(:Base)
-    @worker_connections ||= Rainbows::MODEL_WORKER_CONNECTIONS[@use]
+    super(app, options)
+    defined?(@use) or self.use = Rainbows::Base
+    @worker_connections ||= @use == :Base ? 1 : 50
   end
 
   def reopen_worker_logs(worker_nr)
@@ -33,80 +40,69 @@ class Rainbows::HttpServer < Unicorn::HttpServer
   end
 
   def load_config!
-    use :Base
-    Rainbows.keepalive_timeout = 5
-    Rainbows.max_bytes = 1024 * 1024
-    @worker_connections = nil
     super
-    @worker_connections ||= Rainbows::MODEL_WORKER_CONNECTIONS[@use]
+    @worker_connections = 1 if @use == :Base
+  end
+
+  def worker_loop(worker)
+    Rainbows.forked = true
+    orig = method(:worker_loop)
+    extend(Rainbows.const_get(@use))
+    m = method(:worker_loop)
+    orig == m ? super(worker) : worker_loop(worker)
   end
 
-  def ready_pipe=(v)
-    # hacky hook got force Rainbows! to load modules only in workers
-    if defined?(@master_pid) && @master_pid == Process.ppid
-      extend(Rainbows.const_get(@use))
+  def spawn_missing_workers
+    # 5: std{in,out,err} + heartbeat FD + per-process listener
+    nofile = 5 + @worker_connections + LISTENERS.size
+    trysetrlimit(:RLIMIT_NOFILE, nofile)
+
+    case @use
+    when :ThreadSpawn, :ThreadPool, :ActorSpawn,
+         :CoolioThreadSpawn, :RevThreadSpawn,
+         :XEpollThreadSpawn, :WriterThreadPool, :WriterThreadSpawn
+      trysetrlimit(:RLIMIT_NPROC, @worker_connections + LISTENERS.size + 1)
+    when :XEpollThreadPool, :CoolioThreadPool
+      trysetrlimit(:RLIMIT_NPROC, Rainbows::O[:pool_size] + LISTENERS.size + 1)
     end
     super
   end
 
-  def use(*args)
-    model = args.shift or return @use
-    mod = begin
-      Rainbows.const_get(model)
-    rescue NameError => e
-      logger.error "error loading #{model.inspect}: #{e}"
-      e.backtrace.each { |l| logger.error l }
-      raise ArgumentError, "concurrency model #{model.inspect} not supported"
+  def trysetrlimit(resource, want)
+    var = Process.const_get(resource)
+    cur, max = Process.getrlimit(var)
+    cur <= want and Process.setrlimit(var, cur = max > want ? max : want)
+    if cur == want
+      @logger.warn "#{resource} rlim_cur=#{cur} is barely enough"
+      @logger.warn "#{svc} may monopolize resources dictated by #{resource}" \
+                   " and leave none for your app"
     end
+    rescue => e
+      @logger.error e.message
+      @logger.error "#{resource} needs to be increased to >=#{want} before" \
+                    " starting #{svc}"
+  end
 
-    Module === mod or
-      raise ArgumentError, "concurrency model #{model.inspect} not supported"
-    args.each do |opt|
-      case opt
-      when Hash; Rainbows::O.update(opt)
-      when Symbol; Rainbows::O[opt] = true
-      else; raise ArgumentError, "can't handle option: #{opt.inspect}"
-      end
-    end
-    mod.setup if mod.respond_to?(:setup)
+  def svc
+    File.basename($0)
+  end
+
+  def use=(mod)
+    @use = mod.to_s.split(/::/)[-1].to_sym
     new_defaults = {
-      'rainbows.model' => (@use = model.to_sym),
-      'rack.multithread' => !!(model.to_s =~ /Thread/),
+      'rainbows.model' => @use,
+      'rack.multithread' => !!(mod.to_s =~ /Thread/),
       'rainbows.autochunk' => [:Coolio,:Rev,:Epoll,:XEpoll,
                                :EventMachine,:NeverBlock].include?(@use),
     }
     Rainbows::Const::RACK_DEFAULTS.update(new_defaults)
   end
 
-  def worker_connections(*args)
-    return @worker_connections if args.empty?
-    nr = args[0]
-    (Integer === nr && nr > 0) or
-      raise ArgumentError, "worker_connections must be a positive Integer"
-    @worker_connections = nr
-  end
-
-  def keepalive_timeout(nr)
-    (Integer === nr && nr >= 0) or
-      raise ArgumentError, "keepalive_timeout must be a non-negative Integer"
-    Rainbows.keepalive_timeout = nr
-  end
-
-  def keepalive_requests(nr)
-    Integer === nr or
-      raise ArgumentError, "keepalive_requests must be a non-negative Integer"
+  def keepalive_requests=(nr)
     Unicorn::HttpRequest.keepalive_requests = nr
   end
 
-  def client_max_body_size(nr)
-    err = "client_max_body_size must be nil or a non-negative Integer"
-    case nr
-    when nil
-    when Integer
-      nr >= 0 or raise ArgumentError, err
-    else
-      raise ArgumentError, err
-    end
-    Rainbows.max_bytes = nr
+  def keepalive_requests
+    Unicorn::HttpRequest.keepalive_requests
   end
 end

data/lib/rainbows/join_threads.rb

@@ -5,9 +5,13 @@ module Rainbows::JoinThreads
 
   # blocking acceptor threads must be forced to run
   def self.acceptors(threads)
+    expire = Time.now + Rainbows.server.timeout
     threads.delete_if do |thr|
       Rainbows.tick
       begin
+        # blocking accept() may not wake up properly
+        thr.raise(Errno::EINTR) if Time.now > expire && thr.stop?
+
         thr.run
         thr.join(0.01)
       rescue

data/lib/rainbows/max_body.rb

@@ -6,7 +6,7 @@
 # automatically be configured for you based on the client_max_body_size
 # setting.
 #
-# For more fine-grained conrol, you may also define it per-endpoint in
+# For more fine-grained control, you may also define it per-endpoint in
 # your Rack config.ru like this:
 #
 #        map "/limit_1M" do
@@ -17,16 +17,34 @@
 #          use Rainbows::MaxBody, 1024*1024*10
 #          run MyApp
 #        end
-
+#
+# This is only compatible with concurrency models that expose a streaming
+# "rack.input" to the Rack application.  Thus it is NOT compatible with
+# any of the following as they fully buffer the request body before
+# the application dispatch:
+#
+# * :Coolio
+# * :CoolioThreadPool
+# * :CoolioThreadSpawn
+# * :Epoll
+# * :EventMachine
+# * :NeverBlock
+# * :Rev
+# * :RevThreadPool
+# * :RevThreadSpawn
+# * :XEpoll
+#
+# However, the global Rainbows::Configurator#client_max_body_size is compatible
+# with all concurrency models \Rainbows! supports.
 class Rainbows::MaxBody
 
-
-  # :call-seq:
-  #   # in config.ru:
-  #   use Rainbows::MaxBody, 4096
-  #   run YourApplication.new
-  def initialize(app, limit = Rainbows.max_bytes)
-    Integer === limit or raise ArgumentError, "limit not an Integer"
+  # This is automatically called when used with Rack::Builder#use
+  def initialize(app, limit = nil)
+    case limit
+    when Integer, nil
+    else
+      raise ArgumentError, "limit not an Integer"
+    end
     @app, @limit = app, limit
   end
 
@@ -37,6 +55,7 @@ class Rainbows::MaxBody
 
   # our main Rack middleware endpoint
   def call(env)
+    @limit = Rainbows.server.client_max_body_size if nil == @limit
     catch(:rainbows_EFBIG) do
       len = env[CONTENT_LENGTH]
       if len && len.to_i > @limit
@@ -51,7 +70,7 @@ class Rainbows::MaxBody
   # this is called after forking, so it won't ever affect the master
   # if it's reconfigured
   def self.setup # :nodoc:
-    Rainbows.max_bytes or return
+    Rainbows.server.client_max_body_size or return
     case Rainbows.server.use
     when :Rev, :Coolio, :EventMachine, :NeverBlock,
          :RevThreadSpawn, :RevThreadPool,