rainbows 0.1.1 → 0.2.0

data/lib/rainbows/revactor.rb

@@ -14,12 +14,14 @@ module Rainbows
   # +worker_connections+ will limit the number of client Actors we have
   # running at any one time.
   #
-  # Applications using this model are required to be reentrant, but
-  # generally do not have to worry about race conditions.  Multiple
-  # instances of the same app may run in the same address space
+  # Applications using this model are required to be reentrant, but do
+  # not have to worry about race conditions unless they use threads
+  # internally.  \Rainbows! does not spawn threads under this model.
+  # Multiple instances of the same app may run in the same address space
   # sequentially (but at interleaved points).  Any network dependencies
   # in the application using this model should be implemented using the
-  # \Revactor library as well.
+  # \Revactor library as well, to take advantage of the networking
+  # concurrency features this model provides.
 
   module Revactor
     require 'rainbows/revactor/tee_input'
@@ -32,6 +34,7 @@ module Rainbows
       buf = client.read or return # this probably does not happen...
       hp = HttpParser.new
       env = {}
+      alive = true
       remote_addr = ::Revactor::TCP::Socket === client ?
                     client.remote_addr : LOCALHOST
 
@@ -52,9 +55,10 @@ module Rainbows
           response = app.call(env)
         end
 
-        out = [ hp.keepalive? ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
+        alive = hp.keepalive? && ! Actor.current[:quit]
+        out = [ alive ? CONN_ALIVE : CONN_CLOSE ] if hp.headers?
         HttpResponse.write(client, response, out)
-      end while hp.keepalive? and hp.reset.nil? and env.clear
+      end while alive and hp.reset.nil? and env.clear
       client.close
     # if we get any error, try to write something back to the client
     # assuming we haven't closed the socket, but don't get hung up
@@ -74,59 +78,57 @@ module Rainbows
     # for connections and doesn't die until the parent dies (or is
     # given a INT, QUIT, or TERM signal)
     def worker_loop(worker)
-      ppid = master_pid
       init_worker_process(worker)
-      alive = worker.tmp # tmp is our lifeline to the master process
-
-      trap(:USR1) { reopen_worker_logs(worker.nr) }
-      trap(:QUIT) { alive = false; LISTENERS.each { |s| s.close rescue nil } }
-      [:TERM, :INT].each { |sig| trap(sig) { exit!(0) } } # instant shutdown
 
       root = Actor.current
       root.trap_exit = true
 
       limit = worker_connections
-      listeners = revactorize_listeners
-      logger.info "worker=#{worker.nr} ready with Revactor"
-      clients = 0
+      revactorize_listeners!
+      clients = {}
+      alive = true
 
-      listeners.map! do |s|
+      listeners = LISTENERS.map do |s|
         Actor.spawn(s) do |l|
           begin
-            while clients >= limit
-              logger.info "busy: clients=#{clients} >= limit=#{limit}"
+            while clients.size >= limit
+              logger.info "busy: clients=#{clients.size} >= limit=#{limit}"
               Actor.receive { |filter| filter.when(:resume) {} }
             end
             actor = Actor.spawn(l.accept) { |c| process_client(c) }
-            clients += 1
+            clients[actor.object_id] = actor
             root.link(actor)
           rescue Errno::EAGAIN, Errno::ECONNABORTED
+          rescue Errno::EBADF
+            break
           rescue Object => e
-            if alive
-              logger.error "Unhandled listen loop exception #{e.inspect}."
-              logger.error e.backtrace.join("\n")
-            end
+            listen_loop_error(e) if alive
           end while alive
         end
       end
 
-      nr = 0
+      m = 0
+      check_quit = lambda do
+        worker.tmp.chmod(m = 0 == m ? 1 : 0)
+        if listeners.any? { |l| l.dead? } ||
+           master_pid != Process.ppid ||
+           LISTENERS.first.nil?
+          alive = false
+          clients.each_value { |a| a[:quit] = true }
+        end
+      end
+
       begin
         Actor.receive do |filter|
-          filter.after(1) do
-            if alive
-              alive.chmod(nr = 0 == nr ? 1 : 0)
-              listeners.each { |l| alive = false if l.dead? }
-              ppid == Process.ppid or alive = false
-            end
-          end
+          filter.after(timeout, &check_quit)
           filter.when(Case[:exit, Actor, Object]) do |_,actor,_|
-            orig = clients
-            clients -= 1
+            orig = clients.size
+            clients.delete(actor.object_id)
             orig >= limit and listeners.each { |l| l << :resume }
+            check_quit.call
           end
         end
-      end while alive || clients > 0
+      end while alive || clients.size > 0
     end
 
   private
@@ -141,8 +143,8 @@ module Rainbows
       client.close rescue nil
     end
 
-    def revactorize_listeners
-      LISTENERS.map do |s|
+    def revactorize_listeners!
+      LISTENERS.map! do |s|
         if TCPServer === s
           ::Revactor::TCP.listen(s, nil)
         elsif defined?(::Revactor::UNIX) && UNIXServer === s
@@ -151,7 +153,8 @@ module Rainbows
           logger.error "your version of Revactor can't handle #{s.inspect}"
           nil
         end
-      end.compact
+      end
+      LISTENERS.compact!
     end
 
   end

data/lib/rainbows/thread_pool.rb

@@ -3,8 +3,14 @@
 module Rainbows
 
   # Implements a worker thread pool model.  This is suited for platforms
-  # where the cost of dynamically spawning a new thread for every new
-  # client connection is too high.
+  # like Ruby 1.9, where the cost of dynamically spawning a new thread
+  # for every new client connection is higher than with the ThreadSpawn
+  # model.
+  #
+  # This model should provide a high level of compatibility with all
+  # Ruby implementations, and most libraries and applications.
+  # Applications running under this model should be thread-safe
+  # but not necessarily reentrant.
   #
   # Applications using this model are required to be thread-safe.
   # Threads are never spawned dynamically under this model.  If you're
@@ -12,80 +18,48 @@ module Rainbows
   # consider using the "resolv-replace" library which replaces parts of
   # the core Socket package with concurrent DNS lookup capabilities.
   #
-  # This model is less suited for many slow clients than the others and
-  # thus a lower +worker_connections+ setting is recommended.
+  # This model probably less suited for many slow clients than the
+  # others and thus a lower +worker_connections+ setting is recommended.
+
   module ThreadPool
 
     include Base
 
     def worker_loop(worker)
       init_worker_process(worker)
-      threads = ThreadGroup.new
-      alive = worker.tmp
-      nr = 0
-
-      # closing anything we IO.select on will raise EBADF
-      trap(:USR1) { reopen_worker_logs(worker.nr) rescue nil }
-      trap(:QUIT) { LISTENERS.map! { |s| s.close rescue nil } }
-      [:TERM, :INT].each { |sig| trap(sig) { exit(0) } } # instant shutdown
-      logger.info "worker=#{worker.nr} ready with ThreadPool"
+      RACK_DEFAULTS["rack.multithread"] = true
+      pool = (1..worker_connections).map { new_worker_thread }
+      m = 0
 
       while LISTENERS.first && master_pid == Process.ppid
-        maintain_thread_count(threads)
-        threads.list.each do |thr|
-          alive.chmod(nr += 1)
-          thr.join(timeout / 2.0) and break
+        pool.each do |thr|
+          worker.tmp.chmod(m = 0 == m ? 1 : 0)
+          # if any worker dies, something is serious wrong, bail
+          thr.join(timeout) and break
         end
       end
-      join_worker_threads(threads)
-    end
-
-    def join_worker_threads(threads)
-      logger.info "Joining worker threads..."
-      t0 = Time.now
-      timeleft = timeout
-      threads.list.each { |thr|
-        thr.join(timeleft)
-        timeleft -= (Time.now - t0)
-      }
-      logger.info "Done joining worker threads."
-    end
-
-    def maintain_thread_count(threads)
-      threads.list.each do |thr|
-        next if (Time.now - (thr[:t] || next)) < timeout
-        thr.kill
-        logger.error "killed #{thr.inspect} for being too old"
-      end
-
-      while threads.list.size < worker_connections
-        threads.add(new_worker_thread)
-      end
+      join_threads(pool, worker)
     end
 
     def new_worker_thread
       Thread.new {
         begin
-          ret = begin
-            Thread.current[:t] = Time.now
-            IO.select(LISTENERS, nil, nil, timeout/2.0) or next
+          begin
+            ret = IO.select(LISTENERS, nil, nil, timeout) or next
+            ret.first.each do |sock|
+              begin
+                process_client(sock.accept_nonblock)
+              rescue Errno::EAGAIN, Errno::ECONNABORTED
+              end
+            end
           rescue Errno::EINTR
-            retry
-          rescue Errno::EBADF
+            next
+          rescue Errno::EBADF, TypeError
             return
           end
-          ret.first.each do |sock|
-            begin
-              process_client(sock.accept_nonblock)
-            rescue Errno::EAGAIN, Errno::ECONNABORTED
-            end
-          end
         rescue Object => e
-          if LISTENERS.first
-            logger.error "Unhandled listen loop exception #{e.inspect}."
-            logger.error e.backtrace.join("\n")
-          end
-        end while LISTENERS.first
+          listen_loop_error(e) if LISTENERS.first
+        end while ! Thread.current[:quit] && LISTENERS.first
       }
     end
 

data/lib/rainbows/thread_spawn.rb

@@ -2,75 +2,62 @@
 module Rainbows
 
   # Spawns a new thread for every client connection we accept().  This
-  # model is recommended for platforms where spawning threads is
-  # inexpensive.
+  # model is recommended for platforms like Ruby 1.8 where spawning new
+  # threads is inexpensive.
+  #
+  # This model should provide a high level of compatibility with all
+  # Ruby implementations, and most libraries and applications.
+  # Applications running under this model should be thread-safe
+  # but not necessarily reentrant.
   #
   # If you're connecting to external services and need to perform DNS
   # lookups, consider using the "resolv-replace" library which replaces
   # parts of the core Socket package with concurrent DNS lookup
   # capabilities
+
   module ThreadSpawn
 
     include Base
 
     def worker_loop(worker)
       init_worker_process(worker)
+      RACK_DEFAULTS["rack.multithread"] = true
       threads = ThreadGroup.new
       alive = worker.tmp
-      nr = 0
+      m = 0
       limit = worker_connections
 
-      # closing anything we IO.select on will raise EBADF
-      trap(:USR1) { reopen_worker_logs(worker.nr) rescue nil }
-      trap(:QUIT) { LISTENERS.map! { |s| s.close rescue nil } }
-      [:TERM, :INT].each { |sig| trap(sig) { exit(0) } } # instant shutdown
-      logger.info "worker=#{worker.nr} ready with ThreadSpawn"
-
-      while alive && master_pid == Process.ppid
+      begin
         ret = begin
-          alive.chmod(nr += 1)
-          IO.select(LISTENERS, nil, nil, timeout/2.0) or next
+          alive.chmod(m = 0 == m ? 1 : 0)
+          IO.select(LISTENERS, nil, nil, timeout) or next
         rescue Errno::EINTR
           retry
-        rescue Errno::EBADF
-          alive = false
+        rescue Errno::EBADF, TypeError
+          break
         end
+        alive.chmod(m = 0 == m ? 1 : 0)
 
         ret.first.each do |l|
-          while threads.list.size >= limit
-            nuke_old_thread(threads)
+          # Sleep if we're busy, another less busy worker process may
+          # take it for us if we sleep. This is gross but other options
+          # still suck because they require expensive/complicated
+          # synchronization primitives for _every_ case, not just this
+          # unlikely one.  Since this case is (or should be) uncommon,
+          # just busy wait when we have to.
+          while threads.list.size > limit # unlikely
+            sleep(0.1) # hope another process took it
+            break # back to IO.select
           end
-          c = begin
-            l.accept_nonblock
-          rescue Errno::EINTR, Errno::ECONNABORTED
-            next
+          begin
+            threads.add(Thread.new(l.accept_nonblock) {|c| process_client(c) })
+          rescue Errno::EAGAIN, Errno::ECONNABORTED
           end
-          threads.add(Thread.new(c) { |c| process_client(c) })
         end
-      end
-      join_spawned_threads(threads)
-    end
-
-    def nuke_old_thread(threads)
-      threads.list.each do |thr|
-        next if (Time.now - (thr[:t] || next)) < timeout
-        thr.kill
-        logger.error "killed #{thr.inspect} for being too old"
-        return
-      end
-      # nothing to kill, yield to another thread
-      Thread.pass
-    end
-
-    def join_spawned_threads(threads)
-      logger.info "Joining spawned threads..."
-      t0 = Time.now
-      timeleft = timeout
-      threads.list.each { |thr|
-        thr.join(timeleft)
-        timeleft -= (Time.now - t0)
-      }
-      logger.info "Done joining spawned threads."
+      rescue Object => e
+        listen_loop_error(e) if LISTENERS.first
+      end while LISTENERS.first && master_pid == Process.ppid
+      join_threads(threads.list, worker)
     end
 
   end

data/local.mk.sample

@@ -5,7 +5,7 @@
 # This is depends on a bunch of GNU-isms from bash, sed, touch.
 
 DLEXT := so
-gems := rev-0.3.0 rack-1.0.0 iobuffer-0.1.1
+gems := rev-0.3.1 rack-1.0.0 iobuffer-0.1.1
 
 # Avoid loading rubygems to speed up tests because gmake is
 # fork+exec heavy with Ruby.
@@ -25,8 +25,9 @@ ifdef gem_paths
   RUBYLIB := $(subst $(sp),:,$(addsuffix /lib,$(gem_paths)))
 endif
 
-# pipefail is THE reason to use bash (v3+)
-SHELL := /bin/bash -e -o pipefail
+# pipefail is THE reason to use bash (v3+) or never revisions of ksh93
+# SHELL := /bin/bash -e -o pipefail
+SHELL := /bin/ksh93 -e -o pipefail
 
 full-test: test-18 test-19
 test-18:

data/t/.gitignore

@@ -1,4 +1,3 @@
 /test-results-*
 /test-bin-*
-/*.code
-/*.log
+/random_blob

data/t/GNUmakefile

@@ -21,12 +21,15 @@ T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
 all:: $(T)
 
 # can't rely on "set -o pipefail" since we don't require bash or ksh93 :<
-t_code = $@-$(RUBY_VERSION).code
-t_log = $@-$(RUBY_VERSION).log
-t_run = $(TRACER) $(SHELL) $(TEST_OPTS) $@
+t_pfx = trash/$@-$(RUBY_VERSION)
+t_code = $(t_pfx).code
+t_log = $(t_pfx).log
+# TRACER = strace -f -o $(t_pfx).strace -s 100000
+# TRACER = /usr/bin/time -o $(t_pfx).time
+t_run = $(TRACER) $(SHELL) $(SH_TEST_OPTS) $@
 
 # prefix stdout messages with ':', and stderr messages with '!'
-t_wrap = ( ( ( $(RM) $(t_code); \
+t_wrap = ( ( ( echo 42 > $(t_code); \
   $(t_run); \
   echo $$? > $(t_code) ) \
   | sed 's/^/$(pfx):/' 1>&3 ) 2>&1 \
@@ -38,14 +41,13 @@ ifndef V
   pfx =
 else
   ifeq ($(V),2)
-    TEST_OPTS += -x
+    SH_TEST_OPTS += -x
   endif
   quiet_pre = @echo '* $@';
   quiet_post = 2>&1 | tee $(t_log); exit $$(cat $(t_code))
   pfx = $@
 endif
 
-# TRACER='strace -f -o $@.strace -s 100000'
 run_test = $(quiet_pre) ( $(t_wrap) ) $(quiet_post)
 
 test-bin-$(RUBY_VERSION)/rainbows: ruby_bin = $(shell which $(ruby))
@@ -56,12 +58,24 @@ test-bin-$(RUBY_VERSION)/rainbows: ../bin/rainbows
 	cmp $@+ $@ 2>/dev/null || mv $@+ $@
 	$(RM) $@+
 
+req_random_blob := t3100-revactor-tee-input
+random_blob:
+	dd if=/dev/urandom bs=1M count=10 of=$@+
+	mv $@+ $@
+
+$(addsuffix .sh,$(req_random_blob)): random_blob
+
+$(T): trash/.gitignore
 $(T): export ruby := $(ruby)
 $(T): export PATH := $(CURDIR)/test-bin-$(RUBY_VERSION):$(PATH)
 $(T): test-bin-$(RUBY_VERSION)/rainbows
 	$(run_test)
 
+trash/.gitignore:
+	mkdir -p $(@D)
+	echo '*' > $@
+
 clean:
-	$(RM) -r *.log *.code test-bin-$(RUBY_VERSION)
+	$(RM) -r trash/*.log trash/*.code test-bin-$(RUBY_VERSION)
 
 .PHONY: $(T) clean