rainbows 0.8.0 → 0.9.0

data/Documentation/comparison.haml

@@ -101,7 +101,7 @@
   %li
     rack.input streaming is what makes
     %a(href="http://upr.bogomips.org/") upload progress,
-    BOSH, and Web Sockets possible
+    and BOSH possible
   %li
     rack.input streaming is NOT compatible with current versions of nginx
     or any proxy that fully buffers request bodies before proxying.
@@ -221,78 +221,91 @@
     %th.lock
       %a(href="http://rack.rubyforge.org/doc/Rack/Lock.html") Rack::Lock
     %th.async async
+    %th.ws Web Sockets
   %tr.comp_row
     %td.mod Unicorn/Base
     %td.devfd no-op
     %td.app_pool no-op
     %td.lock no-op
     %td.async lots of RAM :P
+    %td.ws no
   %tr.comp_row
     %td.mod Revactor
     %td.devfd no-op
     %td.app_pool Yes
     %td.lock No!
     %td.async Revactor itself
+    %td.ws Sunshowers
   %tr.comp_row
     %td.mod ThreadPool
     %td.devfd no-op
     %td.app_pool Yes
     %td.lock Yes
     %td.async standard Ruby
+    %td.ws Sunshowers
   %tr.comp_row
     %td.mod Rev
     %td.devfd Yes
     %td.app_pool no-op
     %td.lock no-op
     %td.async DevFdResponse
+    %td.ws no
   %tr.comp_row
     %td.mod ThreadSpawn
     %td.devfd no-op
     %td.app_pool Yes
     %td.lock Yes
     %td.async standard Ruby
+    %td.ws Sunshowers
   %tr.comp_row
     %td.mod EventMachine
     %td.devfd Yes
     %td.app_pool no-op
     %td.lock no-op
     %td.async async_sinatra
+    %td.ws no
   %tr.comp_row
     %td.mod RevThreadSpawn
     %td.devfd Yes
     %td.app_pool Yes
     %td.lock Dumb
     %td.async standard Ruby
+    %td.ws no
   %tr.comp_row
     %td.mod FiberSpawn
     %td.devfd Yes
     %td.app_pool Yes
     %td.lock No!
     %td.async Rainbows::Fiber{::IO,.sleep}
+    %td.ws Sunshowers
   %tr.comp_row
     %td.mod FiberPool
     %td.devfd Yes
     %td.app_pool Yes
     %td.lock No!
     %td.async Rainbows::Fiber{::IO,.sleep}
+    %td.ws Sunshowers
   %tr.comp_row
     %td.mod ActorSpawn
     %td.devfd no-op
     %td.app_pool Yes
     %td.lock Yes
     %td.async standard Ruby
+    %td.ws Sunshowers
   %tr.comp_row
     %td.mod NeverBlock
     %td.devfd Yes
     %td.app_pool Yes*
     %td.lock Yes*
     %td.async NeverBlock, async_sinatra
+    %td.ws no
   %tr.comp_row
     %td.mod RevThreadPool
     %td.devfd Yes
     %td.app_pool Yes
     %td.lock Dumb
     %td.async standard Ruby
+    %td.ws no
 
 %ul
   %li

data/Documentation/rainbows.1.txt

@@ -1,6 +1,6 @@
-% rainbows(1) Unicorn User Manual
-% The Unicorn Community <mongrel-unicorn@rubyforge.org>
-% September 15, 2009
+% rainbows(1) Rainbows! User Manual
+% Rainbows! Hackers <rainbows-talk@rubyforge.org>
+% December 3, 2009
 
 # NAME
 

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.8.0.GIT
+DEF_VER=v0.9.0.GIT
 
 LF='
 '

data/GNUmakefile

@@ -58,7 +58,7 @@ NEWS: GIT-VERSION-FILE
 	$(RAKE) -s news_rdoc > $@+
 	mv $@+ $@
 
-SINCE = 0.6.0
+SINCE = 0.8.0
 ChangeLog: LOG_VERSION = \
   $(shell git rev-parse -q "$(GIT_VERSION)" >/dev/null 2>&1 && \
           echo $(GIT_VERSION) || git describe)

data/README

@@ -22,6 +22,7 @@ For network concurrency, models we currently support are:
 * {FiberSpawn}[link:Rainbows/FiberSpawn.html]
 * {FiberPool}[link:Rainbows/FiberPool.html]
 * {NeverBlock}[link:Rainbows/NeverBlock.html]
+* {RevThreadPool}[link:Rainbows/RevThreadPool.html]
 
 We have {many more on the way}[link:TODO.html] for handling network
 concurrency.  Additionally, we also use multiple processes (managed by
@@ -57,6 +58,7 @@ network concurrency.
 
 \Rainbows is mainly designed for the odd things Unicorn sucks at:
 
+* Web Sockets (via {Sunshowers}[http://rainbows.rubyforge.org/sunshowers/])
 * 3rd-party APIs (to services outside your control/LAN)
 * OpenID consumers (to providers outside your control/LAN)
 * Reverse proxy implementations with editing/censoring
@@ -66,8 +68,7 @@ network concurrency.
 * HTTP server push
 * Long polling
 * Reverse AJAX
-* HTML 5 Web Sockets
-* real-time upload processing
+* real-time upload processing (via {upr}[http://upr.bogomips.org/])
 
 \Rainbows can also be used to service slow clients directly even with
 fast applications.
@@ -115,9 +116,10 @@ command-line switch.  \Rainbows! accepts all options found in
 as well as the "\Rainbows!" block, so you can have the following in your
 config file:
 
+    worker_processes 4 # assuming four CPU cores
     Rainbows! do
-      use :Revactor
-      worker_connections 400
+      use :FiberSpawn
+      worker_connections 100
     end
 
 See the {Rainbows! configuration documentation}[link:Rainbows.html#M000001]

data/Rakefile

@@ -50,7 +50,9 @@ task :news_atom do
           url = "#{cgit_url}/tag/?id=#{tag[:tag]}"
           link! :rel => "alternate", :type => "text/html", :href =>url
           id! url
-          content({:type => 'text'}, tag[:body])
+          message_only = tag[:body].split(/\n.+\(\d+\):\n {6}/s).first.strip
+          content({:type =>:text}, message_only)
+          content(:type =>:xhtml) { pre tag[:body] }
         end
       end
     end
@@ -154,3 +156,32 @@ task :raa_update do
   p res
   puts res.body
 end
+
+desc "post to FM"
+task :fm_update do
+  require 'tempfile'
+  require 'net/http'
+  require 'net/netrc'
+  require 'json'
+  version = ENV['VERSION'] or abort "VERSION= needed"
+  uri = URI.parse('http://freshmeat.net/projects/unicorn/releases.json')
+  rc = Net::Netrc.locate('unicorn-fm') or abort "~/.netrc not found"
+  api_token = rc.password
+  changelog = tags.find { |t| t[:tag] == "v#{version}" }[:body]
+  tmp = Tempfile.new('fm-changelog')
+  tmp.syswrite(changelog)
+  system(ENV["VISUAL"], tmp.path) or abort "#{ENV["VISUAL"]} failed: #$?"
+  changelog = File.read(tmp.path).strip
+
+  req = {
+    "auth_code" => api_token,
+    "release" => {
+      "tag_list" => "Stable",
+      "version" => version,
+      "changelog" => changelog,
+    },
+  }.to_json
+  Net::HTTP.start(uri.host, uri.port) do |http|
+    p http.post(uri.path, req, {'Content-Type'=>'application/json'})
+  end
+end

data/TODO

@@ -9,6 +9,8 @@ care about.
 
 * EventMachine.spawn - should be like Revactor, maybe?
 
+* EventMachine deferrables
+
 * conditional app.deferred?(env) support
   Merb uses it, some other servers support it
 

data/lib/rainbows.rb

@@ -77,11 +77,12 @@ module Rainbows
     :Base => 1, # this one can't change
     :Revactor => 50,
     :ThreadSpawn => 30,
-    :ThreadPool => 10,
+    :ThreadPool => 20,
     :Rev => 50,
     :RevThreadSpawn => 50,
     :RevThreadPool => 50,
     :EventMachine => 50,
+    :EventMachineDefer => 50,
     :FiberSpawn => 50,
     :FiberPool => 50,
     :ActorSpawn => 50,

data/lib/rainbows/base.rb

@@ -43,6 +43,7 @@ module Rainbows
           buf << client.readpartial(CHUNK_SIZE)
         end
 
+        env[CLIENT_IO] = client
         env[RACK_INPUT] = 0 == hp.content_length ?
                  HttpRequest::NULL_IO :
                  Unicorn::TeeInput.new(client, env, hp, buf)

data/lib/rainbows/const.rb

@@ -3,7 +3,7 @@
 module Rainbows
 
   module Const
-    RAINBOWS_VERSION = '0.8.0'
+    RAINBOWS_VERSION = '0.9.0'
 
     include Unicorn::Const
 
@@ -19,5 +19,11 @@ module Rainbows
     CONN_ALIVE = "Connection: keep-alive\r\n"
     LOCALHOST = "127.0.0.1"
 
+    # client IO object that supports reading and writing directly
+    # without filtering it through the HTTP chunk parser.
+    # Maybe we can get this renamed to "rack.io" if it becomes part
+    # of the official spec, but for now it is "hack.io"
+    CLIENT_IO = "hack.io".freeze
+
   end
 end

data/lib/rainbows/error.rb

@@ -19,10 +19,11 @@ module Rainbows
 
       def response(e)
         case e
-        when EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::EINVAL,Errno::EBADF
+        when EOFError, Errno::ECONNRESET,Errno::EPIPE,Errno::EINVAL,Errno::EBADF
           # swallow error if client shuts down one end or disconnects
         when Unicorn::HttpParserError
           Const::ERROR_400_RESPONSE # try to tell the client they're bad
+        when IOError # HttpParserError is an IOError
         else
           app(e)
           Const::ERROR_500_RESPONSE

data/lib/rainbows/ev_core.rb

@@ -8,6 +8,9 @@ module Rainbows
     include Rainbows::Const
     G = Rainbows::G
 
+    # Apps may return this Rack response: AsyncResponse = [ -1, {}, [] ]
+    ASYNC_CALLBACK = "async.callback".freeze
+
     def self.setup(klass)
       klass.const_set(:APP, G.server.app)
     end
@@ -66,7 +69,6 @@ module Rainbows
         if @hp.trailers(@env, @buf << data)
           @input.rewind
           app_call
-          @input.close if File === @input
         end
       end
       rescue => e

data/lib/rainbows/event_machine.rb

@@ -35,9 +35,6 @@ module Rainbows
       include Rainbows::EvCore
       G = Rainbows::G
 
-      # Apps may return this Rack response: AsyncResponse = [ -1, {}, [] ]
-      ASYNC_CALLBACK = 'async.callback'.freeze
-
       def initialize(io)
         @_io = io
       end
@@ -176,7 +173,7 @@ module Rainbows
         return if CUR.size >= MAX
         io = Rainbows.accept(@io) or return
         sig = EM.attach_fd(io.fileno, false)
-        CUR[sig] = Client.new(sig, io)
+        CUR[sig] = CL.new(sig, io)
       end
     end
 
@@ -189,16 +186,18 @@ module Rainbows
       # enable them both, should be non-fatal if not supported
       EM.epoll
       EM.kqueue
-      logger.info "EventMachine: epoll=#{EM.epoll?} kqueue=#{EM.kqueue?}"
+      logger.info "#@use: epoll=#{EM.epoll?} kqueue=#{EM.kqueue?}"
+      client_class = Rainbows.const_get(@use).const_get(:Client)
       Server.const_set(:MAX, worker_connections + LISTENERS.size)
-      EvCore.setup(Client)
+      Server.const_set(:CL, client_class)
+      EvCore.setup(client_class)
       EM.run {
         conns = EM.instance_variable_get(:@conns) or
           raise RuntimeError, "EM @conns instance variable not accessible!"
         Server.const_set(:CUR, conns)
         EM.add_periodic_timer(1) do
           unless G.tick
-            conns.each_value { |client| Client === client and client.quit }
+            conns.each_value { |c| client_class === c and c.quit }
             EM.stop if conns.empty? && EM.reactor_running?
           end
         end

data/lib/rainbows/event_machine_defer.rb

@@ -0,0 +1,59 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# FIXME: fails many tests, experimental
+require 'rainbows/event_machine'
+
+module Rainbows
+
+  # This is currently highly experimental
+  module EventMachineDefer
+    include Rainbows::EventMachine
+
+    class Client < Rainbows::EventMachine::Client
+      undef_method :app_call
+
+      def defer_op
+        @env[RACK_INPUT] = @input
+        @env[REMOTE_ADDR] = @remote_addr
+        @env[ASYNC_CALLBACK] = method(:response_write)
+        catch(:async) { APP.call(@env.update(RACK_DEFAULTS)) }
+        rescue => e
+          handle_error(e)
+          nil
+      end
+
+      def defer_callback(response)
+        # too tricky to support pipelining with :async since the
+        # second (pipelined) request could be a stuck behind a
+        # long-running async response
+        (response.nil? || -1 == response.first) and return @state = :close
+
+        resume
+
+        alive = @hp.keepalive? && G.alive
+        out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if @hp.headers?
+        response_write(response, out, alive)
+        if alive
+          @env.clear
+          @hp.reset
+          @state = :headers
+          if @hp.headers(@env, @buf)
+            EM.next_tick(method(:app_call))
+          else
+            set_comm_inactivity_timeout(G.kato)
+          end
+        else
+          quit
+        end
+      end
+
+      def app_call
+        pause
+        set_comm_inactivity_timeout(0)
+        # defer_callback(defer_op)
+        EM.defer(method(:defer_op), method(:defer_callback))
+      end
+    end
+
+  end
+end

data/lib/rainbows/fiber/base.rb

@@ -82,6 +82,7 @@ module Rainbows
             buf << (client.read_timeout or return)
           end
 
+          env[CLIENT_IO] = client
           env[RACK_INPUT] = 0 == hp.content_length ?
                     HttpRequest::NULL_IO : TeeInput.new(client, env, hp, buf)
           env[REMOTE_ADDR] = remote_addr

data/lib/rainbows/http_server.rb

@@ -61,7 +61,12 @@ module Rainbows
       end
       mod.setup if mod.respond_to?(:setup)
       Const::RACK_DEFAULTS['rainbows.model'] = @use = model.to_sym
-      Const::RACK_DEFAULTS['rack.multithread'] = !!(/Thread/ =~ model.to_s)
+
+      Const::RACK_DEFAULTS['rack.multithread'] = case model.to_s
+      when /Thread/, "EventMachineDefer"; true
+      else false
+      end
+
       case @use
       when :Rev, :EventMachine, :NeverBlock
         Const::RACK_DEFAULTS['rainbows.autochunk'] = true

data/lib/rainbows/never_block.rb

@@ -34,12 +34,14 @@ module Rainbows
       G.server.extend(Core)
     end
 
-    module Client
-
+    module Core
       def self.setup
-        const_set(:POOL, ::NB::Pool::FiberPool.new(O[:pool_size]))
-        Rainbows.const_get(O[:backend]).const_get(:Client).module_eval do
-          include Rainbows::NeverBlock::Client
+        self.const_set(:POOL, ::NB::Pool::FiberPool.new(O[:pool_size]))
+        base = O[:backend].to_s.gsub!(/([a-z])([A-Z])/, '\1_\2').downcase!
+        require "rainbows/never_block/#{base}"
+        Rainbows::NeverBlock.const_get(:Client).class_eval do
+          self.superclass.const_set(:APP, G.server.app)
+          include Rainbows::NeverBlock::Core
           alias _app_call app_call
           undef_method :app_call
           alias app_call nb_app_call
@@ -60,7 +62,7 @@ module Rainbows
     module Core
       def init_worker_process(worker)
         super
-        Client.setup
+        Core.setup
         logger.info "NeverBlock/#{O[:backend]} pool_size=#{O[:pool_size]}"
       end
     end

data/lib/rainbows/never_block/event_machine.rb

@@ -0,0 +1,6 @@
+module Rainbows
+  module NeverBlock
+    class Client < Rainbows::EventMachine::Client
+    end
+  end
+end

data/lib/rainbows/rev/core.rb

@@ -7,7 +7,6 @@ module Rainbows
   module Rev
     class Server < ::Rev::IO
       G = Rainbows::G
-      LOOP = ::Rev::Loop.default
       # CL and MAX will be defined in the corresponding worker loop
 
       def on_readable
@@ -25,10 +24,10 @@ module Rainbows
       def worker_loop(worker)
         init_worker_process(worker)
         mod = self.class.const_get(@use)
+        rloop = Server.const_set(:LOOP, ::Rev::Loop.default)
         Server.const_set(:MAX, @worker_connections)
         Server.const_set(:CL, mod.const_get(:Client))
         EvCore.setup(EvCore)
-        rloop = ::Rev::Loop.default
         Heartbeat.new(1, true).attach(rloop)
         LISTENERS.map! { |s| Server.new(s).attach(rloop) }
         rloop.run

data/lib/rainbows/rev/deferred_response.rb

@@ -10,10 +10,6 @@ module Rainbows
       G = Rainbows::G
       HH = Rack::Utils::HeaderHash
 
-      # we only want to attach to the Rev::Loop belonging to the
-      # main thread in Ruby 1.9
-      LOOP = ::Rev::Loop.default
-
       def self.defer!(client, response, out)
         body = response.last
         headers = HH.new(response[1])
@@ -36,7 +32,9 @@ module Rainbows
             out[0] = CONN_CLOSE
           end
 
-          io = new(io, client, do_chunk, body).attach(LOOP)
+          # we only want to attach to the Rev::Loop belonging to the
+          # main thread in Ruby 1.9
+          io = new(io, client, do_chunk, body).attach(Server::LOOP)
         elsif st.file?
           headers.delete('Transfer-Encoding')
           headers['Content-Length'] ||= st.size.to_s

data/lib/rainbows/rev/master.rb

@@ -1,10 +1,6 @@
 # -*- encoding: binary -*-
 require 'rainbows/rev'
 
-RUBY_VERSION =~ %r{\A1\.8} && ::Rev::VERSION < "0.3.2" and
-  warn "Rainbows::RevThreadSpawn + Rev (< 0.3.2)" \
-       " does not work well under Ruby 1.8"
-
 module Rainbows
 
   module Rev

data/lib/rainbows/rev/thread.rb

@@ -2,6 +2,9 @@
 require 'thread'
 require 'rainbows/rev/master'
 
+RUBY_VERSION =~ %r{\A1\.8} && Rev::VERSION < "0.3.2" and
+  warn "Rev (< 0.3.2) and Threads do not mix well under Ruby 1.8"
+
 module Rainbows
   module Rev
 
@@ -11,7 +14,6 @@ module Rainbows
         KATO.delete(self)
         disable
         @env[RACK_INPUT] = @input
-        @input = nil # not sure why, @input seems to get closed otherwise...
         app_dispatch # must be implemented by subclass
       end
 

data/lib/rainbows/rev_thread_pool.rb

@@ -22,7 +22,7 @@ module Rainbows
   module RevThreadPool
 
     DEFAULTS = {
-      :pool_size => 10, # same default size as ThreadPool (w/o Rev)
+      :pool_size => 20, # same default size as ThreadPool (w/o Rev)
     }
 
     def self.setup

data/lib/rainbows/revactor.rb

@@ -49,6 +49,7 @@ module Rainbows
           buf << client.read(*rd_args)
         end
 
+        env[Const::CLIENT_IO] = client
         env[Const::RACK_INPUT] = 0 == hp.content_length ?
                  HttpRequest::NULL_IO :
                  Rainbows::Revactor::TeeInput.new(client, env, hp, buf)

data/t/simple-http_EventMachineDefer.ru

@@ -0,0 +1,11 @@
+use Rack::ContentLength
+use Rack::ContentType
+run lambda { |env|
+  if env['rack.multithread'] == true &&
+     EM.reactor_running? &&
+     env['rainbows.model'] == :EventMachineDefer
+    [ 200, {}, [ env.inspect << "\n" ] ]
+  else
+    raise "incorrect parameters"
+  end
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rainbows
 version: !ruby/object:Gem::Version 
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors: 
 - Rainbows! hackers
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-02 00:00:00 +00:00
+date: 2009-12-13 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -46,6 +46,7 @@ extra_rdoc_files:
 - lib/rainbows/error.rb
 - lib/rainbows/ev_core.rb
 - lib/rainbows/event_machine.rb
+- lib/rainbows/event_machine_defer.rb
 - lib/rainbows/fiber.rb
 - lib/rainbows/fiber/base.rb
 - lib/rainbows/fiber/io.rb
@@ -55,6 +56,7 @@ extra_rdoc_files:
 - lib/rainbows/http_response.rb
 - lib/rainbows/http_server.rb
 - lib/rainbows/never_block.rb
+- lib/rainbows/never_block/event_machine.rb
 - lib/rainbows/rev.rb
 - lib/rainbows/rev/client.rb
 - lib/rainbows/rev/core.rb
@@ -111,6 +113,7 @@ files:
 - lib/rainbows/error.rb
 - lib/rainbows/ev_core.rb
 - lib/rainbows/event_machine.rb
+- lib/rainbows/event_machine_defer.rb
 - lib/rainbows/fiber.rb
 - lib/rainbows/fiber/base.rb
 - lib/rainbows/fiber/io.rb
@@ -120,6 +123,7 @@ files:
 - lib/rainbows/http_response.rb
 - lib/rainbows/http_server.rb
 - lib/rainbows/never_block.rb
+- lib/rainbows/never_block/event_machine.rb
 - lib/rainbows/rev.rb
 - lib/rainbows/rev/client.rb
 - lib/rainbows/rev/core.rb
@@ -158,6 +162,7 @@ files:
 - t/sha1.ru
 - t/simple-http_Base.ru
 - t/simple-http_EventMachine.ru
+- t/simple-http_EventMachineDefer.ru
 - t/simple-http_FiberPool.ru
 - t/simple-http_FiberSpawn.ru
 - t/simple-http_NeverBlock.ru