rainbows 2.0.1 → 2.1.0

data/Rakefile

@@ -3,105 +3,8 @@ autoload :Gem, 'rubygems'
 autoload :Tempfile, 'tempfile'
 
 # most tasks are in the GNUmakefile which offers better parallelism
-
-def tags
-  timefmt = '%Y-%m-%dT%H:%M:%SZ'
-  @tags ||= `git tag -l`.split(/\n/).map do |tag|
-    if %r{\Av[\d\.]+} =~ tag
-      header, subject, body = `git cat-file tag #{tag}`.split(/\n\n/, 3)
-      header = header.split(/\n/)
-      tagger = header.grep(/\Atagger /).first
-      body ||= "initial"
-      {
-        :time => Time.at(tagger.split(/ /)[-2].to_i).utc.strftime(timefmt),
-        :tagger_name => %r{^tagger ([^<]+)}.match(tagger)[1].strip,
-        :tagger_email => %r{<([^>]+)>}.match(tagger)[1].strip,
-        :id => `git rev-parse refs/tags/#{tag}`.chomp!,
-        :tag => tag,
-        :subject => subject,
-        :body => body,
-      }
-    end
-  end.compact.sort { |a,b| b[:time] <=> a[:time] }
-end
-
 cgit_url = "http://git.bogomips.org/cgit/rainbows.git"
-git_url = ENV['GIT_URL'] || 'git://git.bogomips.org/rainbows.git'
-
-desc 'prints news as an Atom feed'
-task :news_atom do
-  require 'nokogiri'
-  new_tags = tags[0,10]
-  puts(Nokogiri::XML::Builder.new do
-    feed :xmlns => "http://www.w3.org/2005/Atom" do
-      id! "http://rainbows.rubyforge.org/NEWS.atom.xml"
-      title "Rainbows! news"
-      subtitle "Unicorn for sleepy apps and slow clients"
-      link! :rel => 'alternate', :type => 'text/html',
-            :href => 'http://rainbows.rubyforge.org/NEWS.html'
-      updated(new_tags.empty? ? "1970-01-01T00:00:00Z" : new_tags.first[:time])
-      new_tags.each do |tag|
-        entry do
-          title tag[:subject]
-          updated tag[:time]
-          published tag[:time]
-          author {
-            name tag[:tagger_name]
-            email tag[:tagger_email]
-          }
-          url = "#{cgit_url}/tag/?id=#{tag[:tag]}"
-          link! :rel => "alternate", :type => "text/html", :href =>url
-          id! url
-          message_only = tag[:body].split(/\n.+\(\d+\):\n {6}/s).first.strip
-          content({:type =>:text}, message_only)
-          content(:type =>:xhtml) { pre tag[:body] }
-        end
-      end
-    end
-  end.to_xml)
-end
-
-desc 'prints RDoc-formatted news'
-task :news_rdoc do
-  tags.each do |tag|
-    time = tag[:time].tr!('T', ' ').gsub!(/:\d\dZ/, ' UTC')
-    puts "=== #{tag[:tag].sub(/^v/, '')} / #{time}"
-    puts ""
-
-    body = tag[:body]
-    puts tag[:body].gsub(/^/sm, "  ").gsub(/[ \t]+$/sm, "")
-    puts ""
-  end
-end
-
-desc "print release changelog for Rubyforge"
-task :release_changes do
-  version = ENV['VERSION'] or abort "VERSION= needed"
-  version = "v#{version}"
-  vtags = tags.map { |tag| tag[:tag] =~ /\Av/ and tag[:tag] }.sort
-  prev = vtags[vtags.index(version) - 1]
-  if prev
-    system('git', 'diff', '--stat', prev, version) or abort $?
-    puts ""
-    system('git', 'log', "#{prev}..#{version}") or abort $?
-  else
-    system('git', 'log', version) or abort $?
-  end
-end
-
-desc "print release notes for Rubyforge"
-task :release_notes do
-  spec = Gem::Specification.load('rainbows.gemspec')
-  puts spec.description.strip
-  puts ""
-  puts "* #{spec.homepage}"
-  puts "* #{spec.email}"
-  puts "* #{git_url}"
-
-  _, _, body = `git cat-file tag v#{spec.version}`.split(/\n\n/, 3)
-  print "\nChanges:\n\n"
-  puts body
-end
+git_url = 'git://git.bogomips.org/rainbows.git'
 
 desc "read news article from STDIN and post to rubyforge"
 task :publish_news do

data/Summary

@@ -1,7 +0,0 @@
-= \Rainbows! at a glance
-
-Confused by all the options we give you?  So are we!  Here's some tables
-to help keep your head straight.  Remember, engineering is all about
-trade-offs.
-
-INCLUDE

data/TODO

@@ -24,11 +24,11 @@ care about.
   (those who do not require streaming input can use
   {rack-fiber_pool}[http://github.com/mperham/rack-fiber_pool])
 
-* RevFiberPool
+* CoolioFiberPool
 
 * ThreadPoolRevFiber{Spawn,Pool}: just because
 
-* Rev + callcc - current Rev model with callcc (should work with MBARI)
+* Coolio + callcc - current Coolio model with callcc (should work with MBARI)
 
 * Omnibus - haven't looked into it, probably like Revactor with 1.8?
 

data/lib/rainbows/app_pool.rb

@@ -88,7 +88,8 @@ class Rainbows::AppPool < Struct.new(:pool, :re)
     # concurrency models
     self.re ||= begin
       case env["rainbows.model"]
-      when :FiberSpawn, :FiberPool, :Revactor, :NeverBlock, :RevFiberSpawn
+      when :FiberSpawn, :FiberPool, :Revactor, :NeverBlock,
+           :RevFiberSpawn, :CoolioFiberSpawn
         self.pool = Rainbows::Fiber::Queue.new(pool)
       end
       true

data/lib/rainbows/base.rb

@@ -18,6 +18,7 @@ module Rainbows::Base
     super(worker)
     Rainbows::Response.setup(self.class)
     Rainbows::MaxBody.setup
+    Rainbows::RackInput.setup
     G.tmp = worker.tmp
 
     listeners = Rainbows::HttpServer::LISTENERS

data/lib/rainbows/configurator.rb

@@ -13,6 +13,7 @@ module Rainbows::Configurator
   #     worker_connections 400
   #     keepalive_timeout 0 # zero disables keepalives entirely
   #     client_max_body_size 5*1024*1024 # 5 megabytes
+  #     keepalive_requests 666 # default:100
   #   end
   #
   #   # the rest of the Unicorn configuration
@@ -33,6 +34,14 @@ module Rainbows::Configurator
   # The default +client_max_body_size+ is 1 megabyte (1024 * 1024 bytes),
   # setting this to +nil+ will disable body size checks and allow any
   # size to be specified.
+  #
+  # The default +keepalive_requests+ is 100, meaning a client may
+  # complete 100 keepalive requests after the initial request before
+  # \Rainbows! forces a disconnect.  Lowering this can improve
+  # load-balancing characteristics as it forces HTTP/1.1 clients to
+  # reconnect after the specified number of requests, hopefully to a
+  # less busy host or worker process.  This may also be used to mitigate
+  # denial-of-service attacks that use HTTP pipelining.
   def Rainbows!(&block)
     block_given? or raise ArgumentError, "Rainbows! requires a block"
     Rainbows::HttpServer.setup(block)

data/lib/rainbows/const.rb

@@ -2,7 +2,7 @@
 # :enddoc:
 module Rainbows::Const
 
-  RAINBOWS_VERSION = '2.0.1'
+  RAINBOWS_VERSION = '2.1.0'
 
   include Unicorn::Const
 

data/lib/rainbows/coolio/client.rb

@@ -0,0 +1,191 @@
+# -*- encoding: binary -*-
+# :enddoc:
+class Rainbows::Coolio::Client < Coolio::IO
+  include Rainbows::EvCore
+  G = Rainbows::G
+  SF = Rainbows::StreamFile
+  CONN = Rainbows::Coolio::CONN
+  KATO = Rainbows::Coolio::KATO
+  DeferredResponse = Rainbows::Coolio::DeferredResponse
+  DeferredChunkResponse = Rainbows::Coolio::DeferredChunkResponse
+
+  def initialize(io)
+    CONN[self] = false
+    super(io)
+    post_init
+    @deferred = nil
+  end
+
+  def want_more
+    enable unless enabled?
+  end
+
+  def quit
+    super
+    close if @deferred.nil? && @_write_buffer.empty?
+  end
+
+  # override the Coolio::IO#write method try to write directly to the
+  # kernel socket buffers to avoid an extra userspace copy if
+  # possible.
+  def write(buf)
+    if @_write_buffer.empty?
+      begin
+        case rv = @_io.kgio_trywrite(buf)
+        when nil
+          return enable_write_watcher
+        when :wait_writable
+          break # fall through to super(buf)
+        when String
+          buf = rv # retry, skb could grow or been drained
+        end
+      rescue => e
+        return handle_error(e)
+      end while true
+    end
+    super(buf)
+  end
+
+  def on_readable
+    buf = @_io.kgio_tryread(16384)
+    case buf
+    when :wait_readable
+    when nil # eof
+      close
+    else
+      on_read buf
+    end
+  rescue Errno::ECONNRESET
+    close
+  end
+
+  # queued, optional response bodies, it should only be unpollable "fast"
+  # devices where read(2) is uninterruptable.  Unfortunately, NFS and ilk
+  # are also part of this.  We'll also stick DeferredResponse bodies in
+  # here to prevent connections from being closed on us.
+  def defer_body(io)
+    @deferred = io
+    enable_write_watcher
+  end
+
+  # allows enabling of write watcher even when read watcher is disabled
+  def evloop
+    LOOP # this constant is set in when a worker starts
+  end
+
+  def next!
+    attached? or return
+    @deferred = nil
+    enable_write_watcher
+  end
+
+  def timeout?
+    @deferred.nil? && @_write_buffer.empty? and close.nil?
+  end
+
+  # used for streaming sockets and pipes
+  def stream_response(status, headers, io, body)
+    c = stream_response_headers(status, headers) if headers
+    # we only want to attach to the Coolio::Loop belonging to the
+    # main thread in Ruby 1.9
+    io = (c ? DeferredChunkResponse : DeferredResponse).new(io, self, body)
+    defer_body(io.attach(LOOP))
+  end
+
+  def coolio_write_response(response, alive)
+    status, headers, body = response
+    headers = @hp.headers? ? HH.new(headers) : nil
+
+    headers[CONNECTION] = alive ? KEEP_ALIVE : CLOSE if headers
+    if body.respond_to?(:to_path)
+      io = body_to_io(body)
+      st = io.stat
+
+      if st.file?
+        offset, count = 0, st.size
+        if headers
+          if range = make_range!(@env, status, headers)
+            status, offset, count = range
+          end
+          write(response_header(status, headers))
+        end
+        return defer_body(SF.new(offset, count, io, body))
+      elsif st.socket? || st.pipe?
+        return stream_response(status, headers, io, body)
+      end
+      # char or block device... WTF? fall through to body.each
+    end
+    write(response_header(status, headers)) if headers
+    write_body_each(self, body, nil)
+  end
+
+  def app_call
+    KATO.delete(self)
+    @env[RACK_INPUT] = @input
+    @env[REMOTE_ADDR] = @_io.kgio_addr
+    response = APP.call(@env.update(RACK_DEFAULTS))
+
+    coolio_write_response(response, alive = @hp.next? && G.alive)
+    return quit unless alive && :close != @state
+    @state = :headers
+    disable if enabled?
+  end
+
+  def on_write_complete
+    case @deferred
+    when DeferredResponse then return
+    when NilClass # fall through
+    else
+      begin
+        return rev_sendfile(@deferred)
+      rescue EOFError # expected at file EOF
+        close_deferred
+      end
+    end
+
+    case @state
+    when :close
+      close if @_write_buffer.empty?
+    when :headers
+      if @buf.empty?
+        unless enabled?
+          enable
+          KATO[self] = Time.now
+        end
+      else
+        on_read("")
+      end
+    end
+    rescue => e
+      handle_error(e)
+  end
+
+  def handle_error(e)
+    close_deferred
+    if msg = Rainbows::Error.response(e)
+      @_io.kgio_trywrite(msg) rescue nil
+    end
+    @_write_buffer.clear
+    ensure
+      quit
+  end
+
+  def close_deferred
+    case @deferred
+    when DeferredResponse, NilClass
+    else
+      begin
+        @deferred.close
+      rescue => e
+        G.server.logger.error("closing #@deferred: #{e}")
+      end
+      @deferred = nil
+    end
+  end
+
+  def on_close
+    close_deferred
+    CONN.delete(self)
+    KATO.delete(self)
+  end
+end

data/lib/rainbows/coolio/core.rb

@@ -0,0 +1,25 @@
+# -*- encoding: binary -*-
+# :enddoc:
+module Rainbows::Coolio::Core
+  include Rainbows::Base
+
+  # runs inside each forked worker, this sits around and waits
+  # for connections and doesn't die until the parent dies (or is
+  # given a INT, QUIT, or TERM signal)
+  def worker_loop(worker)
+    Rainbows::Response.setup(Rainbows::Coolio::Client)
+    require 'rainbows/coolio/sendfile'
+    Rainbows::Coolio::Client.__send__(:include, Rainbows::Coolio::Sendfile)
+    init_worker_process(worker)
+    mod = Rainbows.const_get(@use)
+    rloop = Rainbows::Coolio::Server.const_set(:LOOP, Coolio::Loop.default)
+    Rainbows::Coolio::Client.const_set(:LOOP, rloop)
+    Rainbows::Coolio::Server.const_set(:MAX, @worker_connections)
+    Rainbows::Coolio::Server.const_set(:CL, mod.const_get(:Client))
+    Rainbows::EvCore.const_set(:APP, G.server.app)
+    Rainbows::EvCore.setup
+    Rainbows::Coolio::Heartbeat.new(1, true).attach(rloop)
+    LISTENERS.map! { |s| Rainbows::Coolio::Server.new(s).attach(rloop) }
+    rloop.run
+  end
+end

data/lib/rainbows/{rev → coolio}/deferred_chunk_response.rb

@@ -1,8 +1,9 @@
 # -*- encoding: binary -*-
 # :enddoc:
 #
-# this is class is specific to Rev for proxying IO-derived objects
-class Rainbows::Rev::DeferredChunkResponse < Rainbows::Rev::DeferredResponse
+# this is class is specific to Coolio for proxying IO-derived objects
+class Rainbows::Coolio::DeferredChunkResponse <
+      Rainbows::Coolio::DeferredResponse
   def on_read(data)
     @client.write("#{data.size.to_s(16)}\r\n")
     @client.write(data)

data/lib/rainbows/{rev → coolio}/deferred_response.rb

@@ -1,9 +1,9 @@
 # -*- encoding: binary -*-
 # :enddoc:
 #
-# this is class is specific to Rev for writing large static files
+# this is class is specific to Coolio for writing large static files
 # or proxying IO-derived objects
-class Rainbows::Rev::DeferredResponse < ::Rev::IO
+class Rainbows::Coolio::DeferredResponse < Coolio::IO
   def initialize(io, client, body)
     super(io)
     @client, @body = client, body
@@ -14,7 +14,7 @@ class Rainbows::Rev::DeferredResponse < ::Rev::IO
   end
 
   def on_close
-    @client.next! if @client.attached? # attached? is false if write fails
     @body.respond_to?(:close) and @body.close
+    @client.next!
   end
 end

data/lib/rainbows/coolio/heartbeat.rb

@@ -0,0 +1,20 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# This class handles the Unicorn fchmod heartbeat mechanism
+# in Coolio-based concurrency models to prevent the master
+# process from killing us unless we're blocked.  This class
+# will also detect and execute the graceful exit if triggered
+# by SIGQUIT
+class Rainbows::Coolio::Heartbeat < Coolio::TimerWatcher
+  KATO = Rainbows::Coolio::KATO
+  CONN = Rainbows::Coolio::CONN
+  G = Rainbows::G
+
+  def on_timer
+    if (ot = G.kato) >= 0
+      ot = Time.now - ot
+      KATO.delete_if { |client, time| time < ot and client.timeout? }
+    end
+    exit if (! G.tick && CONN.size <= 0)
+  end
+end

data/lib/rainbows/{rev → coolio}/master.rb

@@ -1,8 +1,7 @@
 # -*- encoding: binary -*-
 # :enddoc:
-require 'rainbows/rev'
-
-class Rainbows::Rev::Master < Rev::IOWatcher
+require 'thread'
+class Rainbows::Coolio::Master < Coolio::IOWatcher
 
   def initialize(queue)
     @reader, @writer = Kgio::Pipe.new

data/lib/rainbows/{rev → coolio}/sendfile.rb

@@ -1,6 +1,6 @@
 # -*- encoding: binary -*-
 # :enddoc:
-module Rainbows::Rev::Sendfile
+module Rainbows::Coolio::Sendfile
   if IO.method_defined?(:sendfile_nonblock)
     def rev_sendfile(sf) # +sf+ is a Rainbows::StreamFile object
       sf.offset += (n = @_io.sendfile_nonblock(sf, sf.offset, sf.count))

data/lib/rainbows/coolio/server.rb

@@ -0,0 +1,11 @@
+# -*- encoding: binary -*-
+# :enddoc:
+class Rainbows::Coolio::Server < Coolio::IO
+  CONN = Rainbows::Coolio::CONN
+  # CL and MAX will be defined in the corresponding worker loop
+
+  def on_readable
+    return if CONN.size >= MAX
+    io = @_io.kgio_tryaccept and CL.new(io).attach(LOOP)
+  end
+end

data/lib/rainbows/coolio/thread_client.rb

@@ -0,0 +1,36 @@
+# -*- encoding: binary -*-
+# :enddoc:
+
+RUBY_VERSION =~ %r{\A1\.8} and
+  warn "Coolio and Threads do not mix well under Ruby 1.8"
+
+class Rainbows::Coolio::ThreadClient < Rainbows::Coolio::Client
+  def app_call
+    KATO.delete(self)
+    disable if enabled?
+    @env[RACK_INPUT] = @input
+    app_dispatch # must be implemented by subclass
+  end
+
+  # this is only called in the master thread
+  def response_write(response)
+    alive = @hp.next? && G.alive
+    coolio_write_response(response, alive)
+    return quit unless alive && :close != @state
+
+    @state = :headers
+  end
+
+  # fails-safe application dispatch, we absolutely cannot
+  # afford to fail or raise an exception (killing the thread)
+  # here because that could cause a deadlock and we'd leak FDs
+  def app_response
+    begin
+      @env[REMOTE_ADDR] = @_io.kgio_addr
+      APP.call(@env.update(RACK_DEFAULTS))
+    rescue => e
+      Rainbows::Error.app(e) # we guarantee this does not raise
+      [ 500, {}, [] ]
+    end
+  end
+end

data/lib/rainbows/coolio.rb

@@ -0,0 +1,45 @@
+# -*- encoding: binary -*-
+require 'rainbows/coolio_support'
+
+# Implements a basic single-threaded event model with
+# {Cool.io}[http://coolio.github.com/].  It is capable of handling
+# thousands of simultaneous client connections, but with only a
+# single-threaded app dispatch.  It is suited for slow clients and
+# fast applications (applications that do not have slow network
+# dependencies) or applications that use DevFdResponse for deferrable
+# response bodies.  It does not require your Rack application to be
+# thread-safe, reentrancy is only required for the DevFdResponse body
+# generator.
+#
+# Compatibility: Whatever Cool.io itself supports, currently Ruby
+# 1.8/1.9.
+#
+# This model does not implement as streaming "rack.input" which
+# allows the Rack application to process data as it arrives.  This
+# means "rack.input" will be fully buffered in memory or to a
+# temporary file before the application is entered.
+module Rainbows::Coolio
+  # :stopdoc:
+  # keep-alive timeout scoreboard
+  KATO = {}
+
+  # all connected clients
+  CONN = {}
+
+  if {}.respond_to?(:compare_by_identity)
+    CONN.compare_by_identity
+    KATO.compare_by_identity
+  end
+
+  autoload :Master, 'rainbows/coolio/master'
+  autoload :ThreadClient, 'rainbows/coolio/thread_client'
+  autoload :DeferredChunkResponse, 'rainbows/coolio/deferred_chunk_response'
+  # :startdoc:
+end
+# :enddoc:
+require 'rainbows/coolio/heartbeat'
+require 'rainbows/coolio/server'
+require 'rainbows/coolio/core'
+require 'rainbows/coolio/deferred_response'
+require 'rainbows/coolio/client'
+Rainbows::Coolio.__send__ :include, Rainbows::Coolio::Core

data/lib/rainbows/coolio_fiber_spawn.rb

@@ -0,0 +1,26 @@
+# -*- encoding: binary -*-
+require 'rainbows/fiber/coolio'
+
+# A combination of the Coolio and FiberSpawn models.  This allows Ruby
+# 1.9 Fiber-based concurrency for application processing while
+# exposing a synchronous execution model and using scalable network
+# concurrency provided by Cool.io.  A "rack.input" is exposed as well
+# being Sunshowers-compatible.  Applications are strongly advised to
+# wrap all slow IO objects (sockets, pipes) using the
+# Rainbows::Fiber::IO or a Cool.io-compatible class whenever possible.
+module Rainbows::CoolioFiberSpawn
+
+  include Rainbows::Base
+  include Rainbows::Fiber::Coolio
+
+  def worker_loop(worker) # :nodoc:
+    Rainbows::Response.setup(Server)
+    init_worker_process(worker)
+    Server.const_set(:MAX, @worker_connections)
+    Rainbows::Fiber::Base.setup(Server, nil)
+    Server.const_set(:APP, G.server.app)
+    Heartbeat.new(1, true).attach(Coolio::Loop.default)
+    LISTENERS.map! { |s| Server.new(s).attach(Coolio::Loop.default) }
+    Coolio::Loop.default.run
+  end
+end

data/lib/rainbows/coolio_support.rb

@@ -0,0 +1,9 @@
+# -*- encoding: binary -*-
+# :enddoc:
+begin
+  require "coolio"
+  Coolio::VERSION >= "1.0.0" or abort "cool.io >= 1.0.0 is required"
+rescue LoadError
+  require "rev"
+  Rev::VERSION >= "0.3.0" or abort "rev >= 0.3.0 is required"
+end

data/lib/rainbows/coolio_thread_pool/client.rb

@@ -0,0 +1,8 @@
+# -*- encoding: binary -*-
+# :enddoc:
+class Rainbows::CoolioThreadPool::Client < Rainbows::Coolio::ThreadClient
+  # QUEUE constant will be set in worker_loop
+  def app_dispatch
+    QUEUE << self
+  end
+end

data/lib/rainbows/coolio_thread_pool/watcher.rb

@@ -0,0 +1,14 @@
+# -*- encoding: binary -*-
+# :enddoc:
+class Rainbows::CoolioThreadPool::Watcher < Coolio::TimerWatcher
+  G = Rainbows::G
+
+  def initialize(threads)
+    @threads = threads
+    super(G.server.timeout, true)
+  end
+
+  def on_timer
+    @threads.each { |t| t.join(0) and G.quit! }
+  end
+end