rainbows 2.1.0 → 3.0.0

data/lib/rainbows/writer_thread_spawn.rb

@@ -19,19 +19,11 @@ require 'thread'
 # vulnerable to slow client denial-of-service attacks.
 
 module Rainbows::WriterThreadSpawn
-  # :stopdoc:
   include Rainbows::Base
-
-  def write_body(my_sock, body, range) # :nodoc:
-    if body.respond_to?(:close)
-      Rainbows::SyncClose.new(body) { |body| my_sock.queue_body(body, range) }
-    else
-      my_sock.queue_body(body, range)
-    end
-  end
+  autoload :Client, 'rainbows/writer_thread_spawn/client'
 
   def process_client(client) # :nodoc:
-    super(Client.new(client))
+    Client.new(client).process_loop
   end
 
   def worker_loop(worker)  # :nodoc:
@@ -42,4 +34,3 @@ module Rainbows::WriterThreadSpawn
   # :startdoc:
 end
 # :enddoc:
-require 'rainbows/writer_thread_spawn/client'

data/lib/rainbows/writer_thread_spawn/client.rb

@@ -3,17 +3,60 @@
 # used to wrap a BasicSocket to use with +q+ for all writes
 # this is compatible with IO.select
 class Rainbows::WriterThreadSpawn::Client < Struct.new(:to_io, :q, :thr)
-  include Rainbows::Response
   include Rainbows::SocketProxy
+  include Rainbows::ProcessClient
   include Rainbows::WorkerYield
 
   CUR = {} # :nodoc:
 
+  module Methods
+    def write_body_each(body)
+      q << [ :write_body_each, body ]
+    end
+
+    def write_response_close(status, headers, body, alive)
+      to_io.instance_variable_set(:@hp, @hp) # XXX ugh
+      Rainbows::SyncClose.new(body) { |sync_body|
+        q << [ :write_response, status, headers, sync_body, alive ]
+      }
+    end
+
+    if IO.respond_to?(:copy_stream) || IO.method_defined?(:sendfile_nonblock)
+      def write_response(status, headers, body, alive)
+        self.q ||= queue_writer
+        if body.respond_to?(:close)
+          write_response_close(status, headers, body, alive)
+        elsif body.respond_to?(:to_path)
+          write_response_path(status, headers, body, alive)
+        else
+          super
+        end
+      end
+
+      def write_body_file(body, range)
+        q << [ :write_body_file, body, range ]
+      end
+
+      def write_body_stream(body)
+        q << [ :write_body_stream, body ]
+      end
+    else # each-only body response
+      def write_response(status, headers, body, alive)
+        self.q ||= queue_writer
+        if body.respond_to?(:close)
+          write_response_close(status, headers, body, alive)
+        else
+          super
+        end
+      end
+    end # each-only body response
+  end # module Methods
+  include Methods
+
   def self.quit
-    g = Rainbows::G
     CUR.delete_if do |t,q|
       q << nil
-      g.tick
+      Rainbows.tick
       t.alive? ? t.join(0.01) : true
     end until CUR.empty?
   end
@@ -27,16 +70,17 @@ class Rainbows::WriterThreadSpawn::Client < Struct.new(:to_io, :q, :thr)
 
     q = Queue.new
     self.thr = Thread.new(to_io, q) do |io, q|
-      while response = q.shift
+      while op = q.shift
         begin
-          arg1, arg2, arg3 = response
-          case arg1
-          when :body then write_body(io, arg2, arg3)
+          op, *rest = op
+          case op
+          when String
+            io.kgio_write(op)
           when :close
             io.close unless io.closed?
             break
           else
-            io.write(arg1)
+            io.__send__ op, *rest
           end
         rescue => e
           Rainbows::Error.write(io, e)
@@ -51,10 +95,6 @@ class Rainbows::WriterThreadSpawn::Client < Struct.new(:to_io, :q, :thr)
     (self.q ||= queue_writer) << buf
   end
 
-  def queue_body(body, range)
-    (self.q ||= queue_writer) << [ :body, body, range ]
-  end
-
   def close
     if q
       q << :close
@@ -64,6 +104,6 @@ class Rainbows::WriterThreadSpawn::Client < Struct.new(:to_io, :q, :thr)
   end
 
   def closed?
-    false
+    to_io.closed?
   end
 end

data/rainbows.gemspec

@@ -1,14 +1,6 @@
 # -*- encoding: binary -*-
-
 ENV["VERSION"] or abort "VERSION= must be specified"
 manifest = File.readlines('.manifest').map! { |x| x.chomp! }
-
-# don't bother with tests that fork, not worth our time to get working
-# with `gem check -t` ... (of course we care for them when testing with
-# GNU make when they can run in parallel)
-test_files = manifest.grep(%r{\Atest/unit/test_.*\.rb\z}).map do |f|
-  File.readlines(f).grep(/\bfork\b/).empty? ? f : nil
-end.compact
 require 'wrongdoc'
 extend Wrongdoc::Gemspec
 name, summary, title = readme_metadata
@@ -29,16 +21,15 @@ Gem::Specification.new do |s|
   s.rdoc_options = rdoc_options
   s.require_paths = %w(lib)
   s.rubyforge_project = %q{rainbows}
-
-  s.test_files = test_files
+  s.test_files = []
 
   # we want a newer Rack for a valid HeaderHash#each
   s.add_dependency(%q<rack>, ['~> 1.1'])
 
   # we need Unicorn for the HTTP parser and process management
-  s.add_dependency(%q<unicorn>, ["~> 3.2"])
+  s.add_dependency(%q<unicorn>, ["~> 3.3"])
   s.add_development_dependency(%q<isolate>, "~> 3.0.0")
-  s.add_development_dependency(%q<wrongdoc>, "~> 1.0.1")
+  s.add_development_dependency(%q<wrongdoc>, "~> 1.1")
 
   # optional runtime dependencies depending on configuration
   # see t/test_isolate.rb for the exact versions we've tested with

data/t/async_chunk_app.ru

@@ -0,0 +1,62 @@
+# based on async_examples/async_app.ru by James Tucker
+class DeferrableChunkBody
+  include EventMachine::Deferrable
+
+  def call(*body)
+    body.each do |chunk|
+      @body_callback.call("#{chunk.size.to_s(16)}\r\n")
+      @body_callback.call(chunk)
+      @body_callback.call("\r\n")
+    end
+  end
+
+  def each(&block)
+    @body_callback = block
+  end
+
+  def finish
+    @body_callback.call("0\r\n\r\n")
+  end
+end if defined?(EventMachine)
+
+class AsyncChunkApp
+  def call(env)
+    headers = {
+      'Content-Type' => 'text/plain',
+      'Transfer-Encoding' => 'chunked',
+    }
+    delay = env["HTTP_X_DELAY"].to_i
+
+    case env["rainbows.model"]
+    when :EventMachine, :NeverBlock
+      body = DeferrableChunkBody.new
+      body.callback { body.finish }
+      task = lambda {
+        env['async.callback'].call([ 200, headers, body ])
+        EM.add_timer(1) {
+          body.call "Hello "
+
+          EM.add_timer(1) {
+            body.call "World #{env['PATH_INFO']}\n"
+            body.succeed
+          }
+        }
+      }
+      delay == 0 ? EM.next_tick(&task) : EM.add_timer(delay, &task)
+    when :Coolio
+      # Cool.io only does one-shot responses due to the lack of the
+      # equivalent of EM::Deferrables
+      body = [ "Hello ", "World #{env['PATH_INFO']}\n", '' ].map do |chunk|
+        "#{chunk.size.to_s(16)}\r\n#{chunk}\r\n"
+      end
+
+      next_tick = Coolio::TimerWatcher.new(delay, false)
+      next_tick.on_timer { env['async.callback'].call([ 200, headers, body ]) }
+      next_tick.attach(Coolio::Loop.default)
+    else
+      raise "Not supported: #{env['rainbows.model']}"
+    end
+    nil
+  end
+end
+run AsyncChunkApp.new

data/t/byte-range-common.sh

@@ -0,0 +1,142 @@
+t_begin "byte-range setup vars" && {
+	random_blob_size=$(wc -c < random_blob)
+	rb_1=$(( $random_blob_size - 1 ))
+	range_head=-r-365
+	range_tail=-r155-
+	range_mid=-r200-300
+	range_n1=-r0-$rb_1
+	range_n2=-r0-$(($rb_1 - 1))
+	range_1b_head=-r0-0
+	range_1b_tail=-r$rb_1-$rb_1
+	range_1b_mid=-r200-200
+	range_all=-r0-$random_blob_size
+	url=http://$listen/random_blob
+}
+
+check_content_range () {
+	grep '^< HTTP/1\.1 206 Partial Content' $err
+	grep 'Range:' $err
+	# Content-Range: bytes #{offset}-#{offset+count-1}/#{clen}
+	d='\([0-9]\+\)'
+	start= end= size=
+	eval $(< $err sed -n -e \
+	  "s/^< Content-Range: bytes $d-$d\/$d"'.*$/start=\1 end=\2 size=\3/p')
+	test -n "$start"
+	test -n "$end"
+	test -n "$size"
+
+	# ensure we didn't screw up the sed invocation
+	expect="< Content-Range: bytes $start-$end/$size"
+	test x"$(grep -F "$expect" $err)" = x"$(grep '^< Content-Range:' $err)"
+
+	test $start -le $end
+	test $end -lt $size
+}
+
+t_begin "read random blob sha1s" && {
+	sha1_head=$(curl -sSff $range_head file://random_blob | rsha1)
+	sha1_tail=$(curl -sSff $range_tail file://random_blob | rsha1)
+	sha1_mid=$(curl -sSff $range_mid file://random_blob | rsha1)
+	sha1_n1=$(curl -sSff $range_n1 file://random_blob | rsha1)
+	sha1_n2=$(curl -sSff $range_n2 file://random_blob | rsha1)
+	sha1_1b_head=$(curl -sSff $range_1b_head file://random_blob | rsha1)
+	sha1_1b_tail=$(curl -sSff $range_1b_tail file://random_blob | rsha1)
+	sha1_1b_mid=$(curl -sSff $range_1b_mid file://random_blob | rsha1)
+	sha1_all=$(rsha1 < random_blob)
+	echo "$sha1_all=$sha1_n1"
+}
+
+t_begin "normal full request matches" && {
+	sha1="$(curl -v 2>$err -sSf $url | rsha1)"
+	test x"$sha1_all" = x"$sha1"
+	grep 'Content-Range:' $err && die "Content-Range unexpected"
+	grep 'HTTP/1.1 200 OK' $err || die "200 response expected"
+}
+
+t_begin "crazy offset goes over" && {
+	range_insane=-r$(($random_blob_size * 2))-$(($random_blob_size * 4))
+	curl -vsS 2>$err $range_insane $url >/dev/null
+	grep '^< HTTP/1\.[01] 416 ' $err || die "expected 416 error"
+	grep '^< Content-Range: bytes \*/'$random_blob_size $err || \
+          die "expected Content-Range: bytes */SIZE"
+}
+
+t_begin "keepalive/pipelining is supported on 416 responses" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf 'GET /byte-range-common.sh HTTP/1.1\r\n'
+		printf 'Host: %s\r\n' $listen
+		printf 'Range: bytes=9999999999-9999999999\r\n\r\n'
+		printf 'GET /byte-range-common.sh HTTP/1.1\r\n'
+		printf 'Host: %s\r\n' $listen
+		printf 'Connection: close\r\n'
+		printf 'Range: bytes=0-0\r\n\r\n'
+		wait
+	) | socat - TCP:$listen > $fifo
+
+	< $tmp awk '
+/^HTTP\/1\.1 / && NR == 1 && $2 == 416 { first = $2 }
+/^HTTP\/1\.1 / && NR != 1 && $2 == 206 { second = $2 }
+END { exit((first == 416 && second == 206) ? 0 : 1) }
+	'
+}
+
+t_begin "full request matches with explicit ranges" && {
+	sha1="$(curl -v 2>$err $range_all -sSf $url | rsha1)"
+	check_content_range
+	test x"$sha1_all" = x"$sha1"
+
+	sha1="$(curl -v 2>$err $range_n1 -sSf $url | rsha1)"
+	check_content_range
+	test x"$sha1_all" = x"$sha1"
+
+	range_over=-r0-$(($random_blob_size * 2))
+	sha1="$(curl -v 2>$err $range_over -sSf $url | rsha1)"
+	check_content_range
+	test x"$sha1_all" = x"$sha1"
+}
+
+t_begin "no fence post errors" && {
+	sha1="$(curl -v 2>$err $range_n2 -sSf $url | rsha1)"
+	check_content_range
+	test x"$sha1_n2" = x"$sha1"
+
+	sha1="$(curl -v 2>$err $range_1b_head -sSf $url | rsha1)"
+	check_content_range
+	test x"$sha1_1b_head" = x"$sha1"
+
+	sha1="$(curl -v 2>$err $range_1b_tail -sSf $url | rsha1)"
+	check_content_range
+	test x"$sha1_1b_tail" = x"$sha1"
+
+	sha1="$(curl -v 2>$err $range_1b_mid -sSf $url | rsha1)"
+	check_content_range
+	test x"$sha1_1b_mid" = x"$sha1"
+}
+
+t_begin "head range matches" && {
+	sha1="$(curl -sSfv 2>$err $range_head $url | rsha1)"
+	check_content_range
+	test x"$sha1_head" = x"$sha1"
+}
+
+t_begin "tail range matches" && {
+	sha1="$(curl -sSfv 2>$err $range_tail $url | rsha1)"
+	check_content_range
+	test x"$sha1_tail" = x"$sha1"
+}
+
+t_begin "mid range matches" && {
+	sha1="$(curl -sSfv 2>$err $range_mid $url | rsha1)"
+	check_content_range
+	test x"$sha1_mid" = x"$sha1"
+}
+
+t_begin "shutdown server" && {
+	kill -QUIT $rainbows_pid
+}
+
+t_begin "check stderr" && check_stderr
+
+t_done

data/t/t0022-copy_stream-byte-range.sh

@@ -17,7 +17,7 @@ ThreadSpawn|WriterThreadSpawn|ThreadPool|WriterThreadPool|Base) ;;
 	;;
 esac
 
-t_plan 11 "IO.copy_stream byte range response for $model"
+t_plan 13 "IO.copy_stream byte range response for $model"
 
 t_begin "setup and startup" && {
 	rtmpfiles out err
@@ -25,115 +25,6 @@ t_begin "setup and startup" && {
 	# can't load Rack::Lint here since it clobbers body#to_path
 	rainbows -E none -D large-file-response.ru -c $unicorn_config
 	rainbows_wait_start
-	random_blob_size=$(wc -c < random_blob)
-	rb_1=$(( $random_blob_size - 1 ))
-	range_head=-r-365
-	range_tail=-r155-
-	range_mid=-r200-300
-	range_n1=-r0-$rb_1
-	range_n2=-r0-$(($rb_1 - 1))
-	range_1b_head=-r0-0
-	range_1b_tail=-r$rb_1-$rb_1
-	range_1b_mid=-r200-200
-	range_all=-r0-$random_blob_size
-	url=http://$listen/random_blob
 }
 
-check_content_range () {
-	# Content-Range: bytes #{offset}-#{offset+count-1}/#{clen}
-	awk -F/ -v E=0 -v size=$random_blob_size '
-	$2 == size && /^< Content-Range: bytes [0-9]+-[0-9]+\// {
-		split($1, a, /-/);
-		if (a[1] < size) {
-			E = 0;
-			exit(0);
-		}
-	}
-	END { exit(E) }
-	' < $err
-}
-
-t_begin "read random blob sha1s" && {
-	sha1_head=$(curl -sSff $range_head file://random_blob | rsha1)
-	sha1_tail=$(curl -sSff $range_tail file://random_blob | rsha1)
-	sha1_mid=$(curl -sSff $range_mid file://random_blob | rsha1)
-	sha1_n1=$(curl -sSff $range_n1 file://random_blob | rsha1)
-	sha1_n2=$(curl -sSff $range_n2 file://random_blob | rsha1)
-	sha1_1b_head=$(curl -sSff $range_1b_head file://random_blob | rsha1)
-	sha1_1b_tail=$(curl -sSff $range_1b_tail file://random_blob | rsha1)
-	sha1_1b_mid=$(curl -sSff $range_1b_mid file://random_blob | rsha1)
-	sha1_all=$(rsha1 < random_blob)
-	echo "$sha1_all=$sha1_n1"
-}
-
-t_begin "normal full request matches" && {
-	sha1="$(curl -v 2>$err -sSf $url | rsha1)"
-	test x"$sha1_all" = x"$sha1"
-	grep 'Content-Range:' $err && die "Content-Range unexpected"
-	grep 'HTTP/1.1 200 OK' $err || die "200 response expected"
-}
-
-t_begin "crazy offset goes over" && {
-	range_insane=-r$(($random_blob_size * 2))-$(($random_blob_size * 4))
-	curl -vsS 2>$err $range_insane $url
-	grep 'HTTP/1\.[01] 416 ' $err || die "expected 416 error"
-}
-
-t_begin "full request matches with explicit ranges" && {
-	sha1="$(curl -v 2>$err $range_all -sSf $url | rsha1)"
-	check_content_range
-	test x"$sha1_all" = x"$sha1"
-
-	sha1="$(curl -v 2>$err $range_n1 -sSf $url | rsha1)"
-	check_content_range
-	test x"$sha1_all" = x"$sha1"
-
-	range_over=-r0-$(($random_blob_size * 2))
-	sha1="$(curl -v 2>$err $range_over -sSf $url | rsha1)"
-	check_content_range
-	test x"$sha1_all" = x"$sha1"
-}
-
-t_begin "no fence post errors" && {
-	sha1="$(curl -v 2>$err $range_n2 -sSf $url | rsha1)"
-	check_content_range
-	test x"$sha1_n2" = x"$sha1"
-
-	sha1="$(curl -v 2>$err $range_1b_head -sSf $url | rsha1)"
-	check_content_range
-	test x"$sha1_1b_head" = x"$sha1"
-
-	sha1="$(curl -v 2>$err $range_1b_tail -sSf $url | rsha1)"
-	check_content_range
-	test x"$sha1_1b_tail" = x"$sha1"
-
-	sha1="$(curl -v 2>$err $range_1b_mid -sSf $url | rsha1)"
-	check_content_range
-	test x"$sha1_1b_mid" = x"$sha1"
-}
-
-t_begin "head range matches" && {
-	sha1="$(curl -sSfv $range_head $url | rsha1)"
-	check_content_range
-	test x"$sha1_head" = x"$sha1"
-}
-
-t_begin "tail range matches" && {
-	sha1="$(curl -sSf $range_tail $url | rsha1)"
-	check_content_range
-	test x"$sha1_tail" = x"$sha1"
-}
-
-t_begin "mid range matches" && {
-	sha1="$(curl -sSf $range_mid $url | rsha1)"
-	check_content_range
-	test x"$sha1_mid" = x"$sha1"
-}
-
-t_begin "shutdown server" && {
-	kill -QUIT $rainbows_pid
-}
-
-t_begin "check stderr" && check_stderr
-
-t_done
+. ./byte-range-common.sh