rainbows 3.0.0 → 3.1.0

data/lib/rainbows/thread_pool.rb

@@ -35,7 +35,7 @@ module Rainbows::ThreadPool
         thr.join(1) and Rainbows.quit!
       end
     end
-    join_threads(pool)
+    Rainbows::JoinThreads.acceptors(pool)
   end
 
   def sync_worker # :nodoc:

data/lib/rainbows/writer_thread_pool.rb

@@ -52,7 +52,7 @@ module Rainbows::WriterThreadPool
 
     @@q = qp.map { |q| q.queue }
     super(worker) # accept loop from Unicorn
-    qp.map { |q| q.quit! }
+    qp.each { |q| q.quit! }
   end
   # :startdoc:
 end

data/lib/rainbows/xepoll.rb

@@ -0,0 +1,24 @@
+# -*- encoding: binary -*-
+# :enddoc:
+require 'raindrops'
+require 'rainbows/epoll'
+
+# Edge-triggered epoll concurrency model with blocking accept() in
+# a (hopefully) native thread.  This is recommended over Epoll for
+# Ruby 1.9 users as it can workaround accept()-scalability issues
+# on multicore machines.
+module Rainbows::XEpoll
+  include Rainbows::Base
+  autoload :Client, 'rainbows/xepoll/client'
+
+  def init_worker_process(worker)
+    super
+    Rainbows::Epoll.const_set :EP, SleepyPenguin::Epoll.new
+    Rainbows::Client.__send__ :include, Client
+  end
+
+  def worker_loop(worker) # :nodoc:
+    init_worker_process(worker)
+    Client.run
+  end
+end

data/lib/rainbows/xepoll/client.rb

@@ -0,0 +1,45 @@
+# -*- encoding: binary -*-
+# :enddoc:
+
+module Rainbows::XEpoll::Client
+  N = Raindrops.new(1)
+  Rainbows::Epoll.nr_clients = lambda { N[0] }
+  include Rainbows::Epoll::Client
+  MAX = Rainbows.server.worker_connections
+  THRESH = MAX - 1
+  EP = Rainbows::Epoll::EP
+  THREADS = Rainbows::HttpServer::LISTENERS.map do |sock|
+    Thread.new(sock) do |sock|
+      sleep
+      begin
+        if io = sock.kgio_accept
+          N.incr(0, 1)
+          io.epoll_once
+        end
+        sleep while N[0] >= MAX
+      rescue => e
+        Rainbows::Error.listen_loop(e)
+      end while Rainbows.alive
+    end
+  end
+
+  def self.run
+    THREADS.each { |t| t.run }
+    Rainbows::Epoll.loop
+    Rainbows::JoinThreads.acceptors(THREADS)
+  end
+
+  # only call this once
+  def epoll_once
+    @wr_queue = [] # may contain String, ResponsePipe, and StreamFile objects
+    post_init
+    EP.set(self, IN) # wake up the main thread
+    rescue => e
+      Rainbows::Error.write(self, e)
+  end
+
+  def on_close
+    KATO.delete(self)
+    N.decr(0, 1) == THRESH and THREADS.each { |t| t.run }
+  end
+end

data/pkg.mk

@@ -0,0 +1,171 @@
+RUBY = ruby
+RAKE = rake
+RSYNC = rsync
+WRONGDOC = wrongdoc
+
+GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
+	@./GIT-VERSION-GEN
+-include GIT-VERSION-FILE
+-include local.mk
+DLEXT := $(shell $(RUBY) -rrbconfig -e 'puts RbConfig::CONFIG["DLEXT"]')
+RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
+RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
+lib := lib
+
+ifeq ($(shell test -f script/isolate_for_tests && echo t),t)
+isolate_libs := tmp/isolate/$(RUBY_ENGINE)-$(RUBY_VERSION)/isolate.mk
+$(isolate_libs): script/isolate_for_tests
+	@$(RUBY) script/isolate_for_tests
+-include $(isolate_libs)
+lib := $(lib):$(ISOLATE_LIBS)
+endif
+
+ext := $(firstword $(wildcard ext/*))
+ifneq ($(ext),)
+ext_pfx := tmp/ext/$(RUBY_ENGINE)-$(RUBY_VERSION)
+ext_h := $(wildcard $(ext)/*/*.h $(ext)/*.h)
+ext_src := $(wildcard $(ext)/*.c $(ext_h))
+ext_pfx_src := $(addprefix $(ext_pfx)/,$(ext_src))
+ext_d := $(ext_pfx)/$(ext)/.d
+$(ext)/extconf.rb: $(wildcard $(ext)/*.h)
+	@>> $@
+$(ext_d):
+	@mkdir -p $(@D)
+	@> $@
+$(ext_pfx)/$(ext)/%: $(ext)/% $(ext_d)
+	install -m 644 $< $@
+$(ext_pfx)/$(ext)/Makefile: $(ext)/extconf.rb $(ext_d) $(ext_h)
+	$(RM) -f $(@D)/*.o
+	cd $(@D) && $(RUBY) $(CURDIR)/$(ext)/extconf.rb
+ext_sfx := _ext.$(DLEXT)
+ext_dl := $(ext_pfx)/$(ext)/$(notdir $(ext)_ext.$(DLEXT))
+$(ext_dl): $(ext_src) $(ext_pfx_src) $(ext_pfx)/$(ext)/Makefile
+	@echo $^ == $@
+	$(MAKE) -C $(@D)
+lib := $(lib):$(ext_pfx)/$(ext)
+build: $(ext_dl)
+else
+build:
+endif
+
+pkg_extra += GIT-VERSION-FILE NEWS ChangeLog LATEST
+ChangeLog: GIT-VERSION-FILE .wrongdoc.yml
+	$(WRONGDOC) prepare
+NEWS LATEST: ChangeLog
+
+manifest:
+	$(RM) .manifest
+	$(MAKE) .manifest
+
+.manifest: $(pkg_extra)
+	(git ls-files && for i in $@ $(pkg_extra); do echo $$i; done) | \
+		LC_ALL=C sort > $@+
+	cmp $@+ $@ || mv $@+ $@
+	$(RM) $@+
+
+doc:: .document .wrongdoc.yml $(pkg_extra)
+	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
+	-find ext -type f -name '*.rbc' -exec rm -f '{}' ';'
+	$(RM) -r doc
+	$(WRONGDOC) all
+	install -m644 COPYING doc/COPYING
+	install -m644 $(shell grep '^[A-Z]' .document) doc/
+
+ifneq ($(VERSION),)
+pkggem := pkg/$(rfpackage)-$(VERSION).gem
+pkgtgz := pkg/$(rfpackage)-$(VERSION).tgz
+release_notes := release_notes-$(VERSION)
+release_changes := release_changes-$(VERSION)
+
+release-notes: $(release_notes)
+release-changes: $(release_changes)
+$(release_changes):
+	$(WRONGDOC) release_changes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+$(release_notes):
+	$(WRONGDOC) release_notes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+
+# ensures we're actually on the tagged $(VERSION), only used for release
+verify:
+	test x"$(shell umask)" = x0022
+	git rev-parse --verify refs/tags/v$(VERSION)^{}
+	git diff-index --quiet HEAD^0
+	test $$(git rev-parse --verify HEAD^0) = \
+	     $$(git rev-parse --verify refs/tags/v$(VERSION)^{})
+
+fix-perms:
+	-git ls-tree -r HEAD | awk '/^100644 / {print $$NF}' | xargs chmod 644
+	-git ls-tree -r HEAD | awk '/^100755 / {print $$NF}' | xargs chmod 755
+
+gem: $(pkggem)
+
+install-gem: $(pkggem)
+	gem install $(CURDIR)/$<
+
+$(pkggem): manifest fix-perms
+	gem build $(rfpackage).gemspec
+	mkdir -p pkg
+	mv $(@F) $@
+
+$(pkgtgz): distdir = $(basename $@)
+$(pkgtgz): HEAD = v$(VERSION)
+$(pkgtgz): manifest fix-perms
+	@test -n "$(distdir)"
+	$(RM) -r $(distdir)
+	mkdir -p $(distdir)
+	tar cf - $$(cat .manifest) | (cd $(distdir) && tar xf -)
+	cd pkg && tar cf - $(basename $(@F)) | gzip -9 > $(@F)+
+	mv $@+ $@
+
+package: $(pkgtgz) $(pkggem)
+
+test-release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	@echo rubyforge add_release -f \
+	  -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	@echo gem push $(pkggem)
+	@echo rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	rubyforge add_release -f -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	# push gem to RubyGems.org
+	gem push $(pkggem)
+	# in case of gem downloads from RubyForge releases page
+	rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+else
+gem install-gem: GIT-VERSION-FILE
+	$(MAKE) $@ VERSION=$(GIT_VERSION)
+endif
+
+all:: test
+test_units := $(wildcard test/test_*.rb)
+test: test-unit
+test-unit: $(test_units)
+$(test_units): build
+	$(RUBY) -I $(lib) $@ $(RUBY_TEST_OPTS)
+
+# this requires GNU coreutils variants
+ifneq ($(RSYNC_DEST),)
+publish_doc:
+	-git set-file-times
+	$(MAKE) doc
+	find doc/images -type f | \
+		TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css
+	$(MAKE) doc_gz
+	$(RSYNC) -av doc/ $(RSYNC_DEST)/
+	git ls-files | xargs touch
+endif
+
+# Create gzip variants of the same timestamp as the original so nginx
+# "gzip_static on" can serve the gzipped versions directly.
+doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
+doc_gz:
+	for i in $(docs); do \
+	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
+
+.PHONY: all .FORCE-GIT-VERSION-FILE doc test $(test_units) manifest

data/rainbows.gemspec

@@ -19,17 +19,15 @@ Gem::Specification.new do |s|
   s.homepage = Wrongdoc.config[:rdoc_url]
   s.summary = summary
   s.rdoc_options = rdoc_options
-  s.require_paths = %w(lib)
   s.rubyforge_project = %q{rainbows}
-  s.test_files = []
 
   # we want a newer Rack for a valid HeaderHash#each
   s.add_dependency(%q<rack>, ['~> 1.1'])
 
   # we need Unicorn for the HTTP parser and process management
-  s.add_dependency(%q<unicorn>, ["~> 3.3"])
+  s.add_dependency(%q<unicorn>, ["~> 3.4"])
   s.add_development_dependency(%q<isolate>, "~> 3.0.0")
-  s.add_development_dependency(%q<wrongdoc>, "~> 1.1")
+  s.add_development_dependency(%q<wrongdoc>, "~> 1.5")
 
   # optional runtime dependencies depending on configuration
   # see t/test_isolate.rb for the exact versions we've tested with

data/t/GNUmakefile

@@ -19,6 +19,10 @@ endif
 RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
 export RUBY_VERSION RUBY_ENGINE
 
+ifeq (Linux,$(shell uname -s))
+  models += XEpoll
+  models += Epoll
+endif
 models += WriterThreadPool
 models += WriterThreadSpawn
 models += ThreadPool

data/t/bin/content-md5-put

@@ -15,6 +15,7 @@ if ARGV.grep("--no-headers").empty?
   $stdout.write(
       "PUT / HTTP/1.1\r\n" \
       "Host: example.com\r\n" \
+      "Connection: #{ENV["Connection"] || 'close'}\r\n" \
       "Transfer-Encoding: chunked\r\n" \
       "Trailer: Content-MD5\r\n" \
       "\r\n"

data/t/kgio-pipe-response.ru

@@ -1,10 +1,18 @@
 # must be run without Rack::Lint since that clobbers to_path
 use Rainbows::DevFdResponse
 run(lambda { |env|
+  io = case env["rainbows.model"].to_s
+  when /Fiber/
+    Rainbows::Fiber::IO::Pipe
+  else
+    Kgio::Pipe
+  end.popen('cat random_blob', 'rb')
+
   [ 200,
     {
       'Content-Length' => ::File.stat('random_blob').size.to_s,
       'Content-Type' => 'application/octet-stream',
     },
-    Rainbows::Fiber::IO::Pipe.popen('cat random_blob', 'rb') ]
+    io
+  ]
 })

data/t/rack-fiber_pool/app.ru

@@ -2,4 +2,9 @@ require 'rack/fiber_pool'
 use Rack::FiberPool
 use Rack::ContentLength
 use Rack::ContentType, 'text/plain'
-run lambda { |env| [ 200, {}, [ "#{Fiber.current}\n" ] ] }
+run lambda { |env|
+  f = Fiber.current
+  EM.add_timer(3) { f.resume }
+  Fiber.yield
+  [ 200, {}, [ "#{f}\n" ] ]
+}

data/t/simple-http_Epoll.ru

@@ -0,0 +1,9 @@
+use Rack::ContentLength
+use Rack::ContentType
+run lambda { |env|
+  if env['rack.multithread'] == false && env['rainbows.model'] == :Epoll
+    [ 200, {}, [ Thread.current.inspect << "\n" ] ]
+  else
+    raise env.inspect
+  end
+}

data/t/simple-http_XEpoll.ru

@@ -0,0 +1,9 @@
+use Rack::ContentLength
+use Rack::ContentType
+run lambda { |env|
+  if env['rack.multithread'] == false && env['rainbows.model'] == :XEpoll
+    [ 200, {}, [ Thread.current.inspect << "\n" ] ]
+  else
+    raise env.inspect
+  end
+}

data/t/t0014-config-conflict.sh

@@ -4,7 +4,7 @@ t_plan 6 "config variables conflict with preload_app"
 
 t_begin "setup and start" && {
 	rainbows_setup
-	rtmpfiles ru rutmp
+	rtmpfiles ru
 
 	cat > $ru <<\EOF
 use Rack::ContentLength
@@ -23,8 +23,10 @@ t_begin "hit with curl" && {
 }
 
 t_begin "modify rackup file" && {
-	sed -e 's/world/WORLD/' < $ru > $rutmp
-	mv $rutmp $ru
+	ed -s $ru <<EOF
+,s/world/WORLD/
+w
+EOF
 }
 
 t_begin "reload signal succeeds" && {

data/t/t0023-sendfile-byte-range.sh

@@ -9,13 +9,7 @@ ruby) ;;
 	;;
 esac
 
-case $model in
-EventMachine|NeverBlock)
-	t_info "skipping $T since it's not compatible with $model"
-	exit 0
-	;;
-*) ;;
-esac
+skip_models EventMachine NeverBlock
 
 t_plan 13 "sendfile byte range response for $model"
 

data/t/t0034-pipelined-pipe-response.sh

@@ -22,12 +22,13 @@ require "kcar"
 $stdin.binmode
 expect = ENV["random_blob_sha1"]
 kcar = Kcar::Response.new($stdin, {})
-3.times do
+3.times do |i|
 	nr = 0
 	status, headers, body = kcar.rack
 	dig = Digest::SHA1.new
 	body.each { |buf| dig << buf ; nr += buf.size }
 	sha1 = dig.hexdigest
+	warn "[#{i}] nr: #{nr}"
 	sha1 == expect or abort "mismatch: sha1=#{sha1} != expect=#{expect}"
 	body.close
 end

data/t/t0035-kgio-pipe-response.sh

@@ -1,13 +1,6 @@
 #!/bin/sh
 . ./test-lib.sh
 test -r random_blob || die "random_blob required, run with 'make $0'"
-case $model in
-*Fiber* ) ;;
-*)
-	t_info "skipping $T since it's not compatible with $model"
-	exit 0
-	;;
-esac
 
 t_plan 10 "fast Kgio pipe response for $model"
 

data/t/t0041-optional-pool-size.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+. ./test-lib.sh
+
+case $model in
+NeverBlock|CoolioThreadPool) ;;
+*)
+	t_info "skipping $T since it doesn't support :pool_size"
+	exit
+	;;
+esac
+
+t_plan 6 "optional :pool_size argument for $model"
+
+t_begin "setup and startup" && {
+	rtmpfiles curl_out curl_err
+	rainbows_setup $model
+}
+
+t_begin "fails with bad :pool_size" && {
+	ed -s $unicorn_config <<EOF
+,s/use :$model/use :$model, :pool_size => -666/
+w
+EOF
+	grep "pool_size" $unicorn_config
+	rainbows -D env.ru -c $unicorn_config || echo err=$? > $ok
+	test x"$(cat $ok)" = "xerr=1"
+}
+
+t_begin "starts with correct :pool_size" && {
+	ed -s $unicorn_config <<EOF
+,s/use :$model.*/use :$model, :pool_size => 6/
+w
+EOF
+	grep "pool_size" $unicorn_config
+	rainbows -D env.ru -c $unicorn_config
+	rainbows_wait_start
+}
+
+t_begin "regular TCP request works right" && {
+	curl -sSfv http://$listen/
+}
+
+t_begin "no errors in stderr" && {
+	check_stderr
+}
+
+t_begin "shutdown" && {
+	kill $rainbows_pid
+}
+
+t_done