rainbows 0.3.0 → 0.4.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.3.0.GIT
+DEF_VER=v0.4.0.GIT
 
 LF='
 '

data/GNUmakefile

@@ -1,6 +1,6 @@
 # use GNU Make to run tests in parallel, and without depending on Rubygems
 all::
-ruby = ruby
+RUBY = ruby
 rake = rake
 GIT_URL = git://git.bogomips.org/rainbows.git
 
@@ -8,11 +8,17 @@ GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
 	@./GIT-VERSION-GEN
 -include GIT-VERSION-FILE
 -include local.mk
+ifdef ruby
+  ifeq ($(RUBY),ruby)
+    $(warning ruby variable is deprecated, use RUBY instead)
+    RUBY = $(ruby)
+  endif
+endif
 ifeq ($(DLEXT),) # "so" for Linux
-  DLEXT := $(shell $(ruby) -rrbconfig -e 'puts Config::CONFIG["DLEXT"]')
+  DLEXT := $(shell $(RUBY) -rrbconfig -e 'puts Config::CONFIG["DLEXT"]')
 endif
 ifeq ($(RUBY_VERSION),)
-  RUBY_VERSION := $(shell $(ruby) -e 'puts RUBY_VERSION')
+  RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
 endif
 
 base_bins := rainbows
@@ -25,7 +31,7 @@ install: $(bins)
 	$(RM) -r .install-tmp
 	mkdir .install-tmp
 	cp -p bin/* .install-tmp
-	$(ruby) setup.rb all
+	$(RUBY) setup.rb all
 	$(RM) $^
 	mv .install-tmp/* bin/
 	$(RM) -r .install-tmp
@@ -82,10 +88,10 @@ doc: .document NEWS ChangeLog
 	cd doc && for i in $(base_bins); do \
 	  sed -e '/"documentation">/r man1/'$$i'.1.html' \
 		< $${i}_1.html > tmp && mv tmp $${i}_1.html; done
-	$(ruby) -i -p -e \
+	$(RUBY) -i -p -e \
 	  '$$_.gsub!("</title>",%q{\&$(call atom,$(cgit_atom))})' \
 	  doc/ChangeLog.html
-	$(ruby) -i -p -e \
+	$(RUBY) -i -p -e \
 	  '$$_.gsub!("</title>",%q{\&$(call atom,$(news_atom))})' \
 	  doc/NEWS.html doc/README.html
 	$(rake) -s news_atom > doc/NEWS.atom.xml

data/README

@@ -13,10 +13,11 @@ suck; differently.
 
 For network concurrency, models we currently support are:
 
-* {:ThreadSpawn}[link:Rainbows/ThreadSpawn.html]
-* {:ThreadPool}[link:Rainbows/ThreadPool.html]
 * {:Revactor}[link:Rainbows/Revactor.html]
+* {:ThreadPool}[link:Rainbows/ThreadPool.html]
 * {:Rev}[link:Rainbows/Rev.html]*
+* {:ThreadSpawn}[link:Rainbows/ThreadSpawn.html]
+* {:EventMachine}[link:Rainbows/EventMachine.html]
 
 We have {more on the way}[link:TODO.html] for handling network concurrency.
 Additionally, we also use multiple processes (managed by Unicorn) for

data/Rakefile

@@ -12,8 +12,8 @@ def tags
       body ||= "initial"
       {
         :time => Time.at(tagger.split(/ /)[-2].to_i).utc.strftime(timefmt),
-        :tagger_name => %r{^tagger ([^<]+)}.match(tagger)[1],
-        :tagger_email => %r{<([^>]+)>}.match(tagger)[1],
+        :tagger_name => %r{^tagger ([^<]+)}.match(tagger)[1].strip,
+        :tagger_email => %r{<([^>]+)>}.match(tagger)[1].strip,
         :id => `git rev-parse refs/tags/#{tag}`.chomp!,
         :tag => tag,
         :subject => subject,
@@ -49,7 +49,7 @@ task :news_atom do
           url = "#{cgit_url}/tag/?id=#{tag[:tag]}"
           link! :rel => "alternate", :type => "text/html", :href =>url
           id! url
-          content(:type => 'text') { tag[:body] }
+          content({:type => 'text'}, tag[:body])
         end
       end
     end

data/TODO

@@ -7,18 +7,9 @@ care about.
   unit tests, only integration tests that exercise externally
   visible parts.
 
-* (maybe) make our tests TAP-compatible.  Unfortunately the the
-  only shell library we've seen for TAP seems to use bashisms.
-
 * Rev + Thread - current Rev model with threading, which will give
   us a streaming (but rewindable) "rack.input".
 
-* EventMachine - much like Rev, but we haven't looked at this one much
-  (our benevolent dictator doesn't like C++).  If we can figure out how
-  to do Rev without Revactor, then this should be pretty easy.
-
-* EventMachine + catch/throw :async API used in Thin
-
 * EventMachine.spawn - should be like Revactor, maybe?
 
 * Rev + callcc - current Rev model with callcc (should work with MBARI)
@@ -28,6 +19,8 @@ care about.
 
 * Omnibus - haven't looked into it, probably like Revactor with 1.8?
 
+* Packet - pure Ruby, EventMachine-like library
+
 * Rubinius Actors - should be like Revactor and easily doable once
   Rubinius gets more mature.
 

data/lib/rainbows.rb

@@ -56,6 +56,7 @@ module Rainbows
     :ThreadSpawn => 30,
     :ThreadPool => 10,
     :Rev => 50,
+    :EventMachine => 50,
   }.each do |model, _|
     u = model.to_s.gsub(/([a-z0-9])([A-Z0-9])/) { "#{$1}_#{$2.downcase!}" }
     autoload model, "rainbows/#{u.downcase!}"

data/lib/rainbows/app_pool.rb

@@ -39,14 +39,18 @@ module Rainbows
   # AppPool should be used if you want to enforce a lower value of +P+
   # than +N+.
   #
-  # AppPool has no effect on the Rev concurrency model as that is
-  # single-threaded/single-instance as far as application concurrency goes.
-  # In other words, +P+ is always +one+ when using Rev.  AppPool currently
-  # only works with the ThreadSpawn and ThreadPool models.  It does not
-  # yet work reliably with the Revactor model, yet.
+  # AppPool has no effect on the Rev or EventMachine concurrency models
+  # as those are single-threaded/single-instance as far as application
+  # concurrency goes.  In other words, +P+ is always +one+ when using
+  # Rev or EventMachine.  AppPool currently only works with the
+  # ThreadSpawn and ThreadPool models.  It does not yet work reliably
+  # with the Revactor model, but actors are far more lightweight and
+  # probably better suited for lightweight applications that would
+  # not benefit from AppPool.
   #
   # Since this is Rack middleware, you may load this in your Rack
-  # config.ru file and even use it in servers other than \Rainbows!
+  # config.ru file and even use it in threaded servers other than
+  # \Rainbows!
   #
   #   use Rainbows::AppPool, :size => 30
   #   map "/lobster" do

data/lib/rainbows/base.rb

@@ -93,7 +93,7 @@ module Rainbows
       Rainbows::G.alive = false
       expire = Time.now + (timeout * 2.0)
       m = 0
-      while (nr = threads.count { |thr| thr.alive? }) > 0
+      until (threads.delete_if { |thr| ! thr.alive? }).empty?
         threads.each { |thr|
           worker.tmp.chmod(m = 0 == m ? 1 : 0)
           thr.join(1)

data/lib/rainbows/const.rb

@@ -3,7 +3,7 @@
 module Rainbows
 
   module Const
-    RAINBOWS_VERSION = '0.3.0'
+    RAINBOWS_VERSION = '0.4.0'
 
     include Unicorn::Const
 

data/lib/rainbows/ev_core.rb

@@ -0,0 +1,88 @@
+# -*- encoding: binary -*-
+
+module Rainbows
+
+  # base module for evented models like Rev and EventMachine
+  module EvCore
+    include Unicorn
+    include Rainbows::Const
+    G = Rainbows::G
+
+    def post_init
+      @remote_addr = ::TCPSocket === @_io ? @_io.peeraddr.last : LOCALHOST
+      @env = {}
+      @hp = HttpParser.new
+      @state = :headers # [ :body [ :trailers ] ] :app_call :close
+      @buf = ""
+      @deferred_bodies = [] # for (fast) regular files only
+    end
+
+    # graceful exit, like SIGQUIT
+    def quit
+      @state = :close
+    end
+
+    def handle_error(e)
+      msg = case e
+      when EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::EINVAL,Errno::EBADF
+        ERROR_500_RESPONSE
+      when HttpParserError # try to tell the client they're bad
+        ERROR_400_RESPONSE
+      else
+        G.logger.error "Read error: #{e.inspect}"
+        G.logger.error e.backtrace.join("\n")
+        ERROR_500_RESPONSE
+      end
+      write(msg)
+      quit
+    end
+
+    def tmpio
+      io = Util.tmpio
+      def io.size
+        # already sync=true at creation, so no need to flush before stat
+        stat.size
+      end
+      io
+    end
+
+    # TeeInput doesn't map too well to this right now...
+    def on_read(data)
+      case @state
+      when :headers
+        @hp.headers(@env, @buf << data) or return
+        @state = :body
+        len = @hp.content_length
+        if len == 0
+          @input = HttpRequest::NULL_IO
+          app_call # common case
+        else # nil or len > 0
+          # since we don't do streaming input, we have no choice but
+          # to take over 100-continue handling from the Rack application
+          if @env[HTTP_EXPECT] =~ /\A100-continue\z/i
+            write(EXPECT_100_RESPONSE)
+            @env.delete(HTTP_EXPECT)
+          end
+          @input = len && len <= MAX_BODY ? StringIO.new("") : tmpio
+          @hp.filter_body(@buf2 = @buf.dup, @buf)
+          @input << @buf2
+          on_read("")
+        end
+      when :body
+        if @hp.body_eof?
+          @state = :trailers
+          on_read(data)
+        elsif data.size > 0
+          @hp.filter_body(@buf2, @buf << data)
+          @input << @buf2
+          on_read("")
+        end
+      when :trailers
+        @hp.trailers(@env, @buf << data) and app_call
+      end
+      rescue Object => e
+        handle_error(e)
+    end
+
+  end
+end

data/lib/rainbows/event_machine.rb

@@ -0,0 +1,218 @@
+# -*- encoding: binary -*-
+require 'eventmachine'
+EM::VERSION >= '0.12.10' or abort 'eventmachine 0.12.10 is required'
+require 'rainbows/ev_core'
+
+module Rainbows
+
+  # Implements a basic single-threaded event model with
+  # {EventMachine}[http://rubyeventmachine.com/].  It is capable of
+  # handling thousands of simultaneous client connections, but with only
+  # a single-threaded app dispatch.  It is suited for slow clients and
+  # fast applications (applications that do not have slow network
+  # dependencies) or applications that use DevFdResponse for deferrable
+  # response bodies.  It does not require your Rack application to be
+  # thread-safe, reentrancy is only required for the DevFdResponse body
+  # generator.
+  #
+  # Compatibility: Whatever \EventMachine ~> 0.12.10 and Unicorn both
+  # support, currently Ruby 1.8/1.9.
+  #
+  # This model is compatible with users of "async.callback" in the Rack
+  # environment such as
+  # {async_sinatra}[http://github.com/raggi/async_sinatra].
+  #
+  # This model does not implement as streaming "rack.input" which allows
+  # the Rack application to process data as it arrives.  This means
+  # "rack.input" will be fully buffered in memory or to a temporary file
+  # before the application is entered.
+
+  module EventMachine
+
+    include Base
+
+    class Client < EM::Connection
+      include Rainbows::EvCore
+      G = Rainbows::G
+
+      # Apps may return this Rack response: AsyncResponse = [ -1, {}, [] ]
+      ASYNC_CALLBACK = 'async.callback'.freeze
+
+      def initialize(io)
+        @_io = io
+      end
+
+      alias write send_data
+      alias receive_data on_read
+
+      def quit
+        super
+        close_connection_after_writing
+      end
+
+      def app_call
+        begin
+          (@env[RACK_INPUT] = @input).rewind
+          alive = @hp.keepalive?
+          @env[REMOTE_ADDR] = @remote_addr
+          @env[ASYNC_CALLBACK] = method(:response_write)
+
+          response = catch(:async) { G.app.call(@env.update(RACK_DEFAULTS)) }
+
+          # too tricky to support pipelining with :async since the
+          # second (pipelined) request could be a stuck behind a
+          # long-running async response
+          (response.nil? || -1 == response.first) and return @state = :close
+
+          alive &&= G.alive
+          out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if @hp.headers?
+          response_write(response, out, alive)
+
+          if alive
+            @env.clear
+            @hp.reset
+            @state = :headers
+            # keepalive requests are always body-less, so @input is unchanged
+            @hp.headers(@env, @buf) and next
+          end
+          return
+        end while true
+      end
+
+      def response_write(response, out = [], alive = false)
+        body = response.last
+        unless body.respond_to?(:to_path)
+          HttpResponse.write(self, response, out)
+          quit unless alive
+          return
+        end
+
+        headers = Rack::Utils::HeaderHash.new(response[1])
+        path = body.to_path
+        io = body.to_io if body.respond_to?(:to_io)
+        io ||= IO.new($1.to_i) if path =~ %r{\A/dev/fd/(\d+)\z}
+        io ||= File.open(path, 'rb') # could be a named pipe
+
+        st = io.stat
+        if st.file?
+          headers.delete('Transfer-Encoding')
+          headers['Content-Length'] ||= st.size.to_s
+          response = [ response.first, headers.to_hash, [] ]
+          HttpResponse.write(self, response, out)
+          stream = stream_file_data(path)
+          stream.callback { quit } unless alive
+        elsif st.socket? || st.pipe?
+          do_chunk = !!(headers['Transfer-Encoding'] =~ %r{\Achunked\z}i)
+          do_chunk = false if headers.delete('X-Rainbows-Autochunk') == 'no'
+          if out.nil?
+            do_chunk = false
+          else
+            out[0] = CONN_CLOSE
+          end
+          response = [ response.first, headers.to_hash, [] ]
+          HttpResponse.write(self, response, out)
+          if do_chunk
+            EM.watch(io, ResponseChunkPipe, self).notify_readable = true
+          else
+            EM.enable_proxy(EM.attach(io, ResponsePipe, self), self)
+          end
+        else
+          HttpResponse.write(self, response, out)
+        end
+      end
+
+      def unbind
+        @_io.close
+      end
+    end
+
+    module ResponsePipe
+      def initialize(client)
+        @client = client
+      end
+
+      def unbind
+        @io.close
+        @client.quit
+      end
+    end
+
+    module ResponseChunkPipe
+      include ResponsePipe
+
+      def unbind
+        @client.write("0\r\n\r\n")
+        super
+      end
+
+      def notify_readable
+        begin
+          data = begin
+            @io.read_nonblock(16384)
+          rescue Errno::EINTR
+            retry
+          rescue Errno::EAGAIN
+            return
+          rescue EOFError
+            detach
+            return
+          end
+          @client.send_data(sprintf("%x\r\n", data.size))
+          @client.send_data(data)
+          @client.send_data("\r\n")
+        end while true
+      end
+    end
+
+    module Server
+
+      def initialize(conns)
+        @limit = Rainbows::G.max + HttpServer::LISTENERS.size
+        @em_conns = conns
+      end
+
+      def close
+        detach
+        @io.close
+      end
+
+      def notify_readable
+        return if @em_conns.size >= @limit
+        begin
+          io = @io.accept_nonblock
+          sig = EM.attach_fd(io.fileno, false)
+          @em_conns[sig] = Client.new(sig, io)
+        rescue Errno::EAGAIN, Errno::ECONNABORTED
+        end
+      end
+    end
+
+    # runs inside each forked worker, this sits around and waits
+    # for connections and doesn't die until the parent dies (or is
+    # given a INT, QUIT, or TERM signal)
+    def worker_loop(worker)
+      init_worker_process(worker)
+      m = 0
+
+      # enable them both, should be non-fatal if not supported
+      EM.epoll
+      EM.kqueue
+      logger.info "EventMachine: epoll=#{EM.epoll?} kqueue=#{EM.kqueue?}"
+      EM.run {
+        conns = EM.instance_variable_get(:@conns) or
+          raise RuntimeError, "EM @conns instance variable not accessible!"
+        EM.add_periodic_timer(1) do
+          worker.tmp.chmod(m = 0 == m ? 1 : 0)
+          unless G.alive
+            conns.each_value { |client| Client === client and client.quit }
+            EM.stop if conns.empty? && EM.reactor_running?
+          end
+        end
+        LISTENERS.map! do |s|
+          EM.watch(s, Server, conns) { |c| c.notify_readable = true }
+        end
+      }
+    end
+
+  end
+end