unicorn 3.0.1 → 3.1.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v3.0.1.GIT
+DEF_VER=v3.1.0.GIT
 
 LF='
 '

data/lib/unicorn/configurator.rb

@@ -9,6 +9,7 @@ require 'logger'
 # nginx is also available at
 # http://unicorn.bogomips.org/examples/nginx.conf
 class Unicorn::Configurator
+  include Unicorn
   attr_accessor :set, :config_file, :after_reload
 
   # :stopdoc:
@@ -40,6 +41,7 @@ class Unicorn::Configurator
     :pid => nil,
     :preload_app => false,
     :rewindable_input => true, # for Rack 2.x: (Rack::VERSION[0] <= 1),
+    :client_body_buffer_size => Unicorn::Const::MAX_BODY,
   }
   #:startdoc:
 
@@ -180,11 +182,7 @@ class Unicorn::Configurator
   #      server 192.168.0.9:8080 fail_timeout=0;
   #    }
   def timeout(seconds)
-    Numeric === seconds or raise ArgumentError,
-                                "not numeric: timeout=#{seconds.inspect}"
-    seconds >= 3 or raise ArgumentError,
-                                "too low: timeout=#{seconds.inspect}"
-    set[:timeout] = seconds
+    set_int(:timeout, seconds, 3)
   end
 
   # sets the current number of worker_processes to +nr+.  Each worker
@@ -194,11 +192,7 @@ class Unicorn::Configurator
   # the rest of your Unicorn configuration.  See the SIGNALS document
   # for more information.
   def worker_processes(nr)
-    Integer === nr or raise ArgumentError,
-                           "not an integer: worker_processes=#{nr.inspect}"
-    nr >= 0 or raise ArgumentError,
-                           "not non-negative: worker_processes=#{nr.inspect}"
-    set[:worker_processes] = nr
+    set_int(:worker_processes, nr, 1)
   end
 
   # sets listeners to the given +addresses+, replacing or augmenting the
@@ -392,6 +386,14 @@ class Unicorn::Configurator
     set_bool(:rewindable_input, bool)
   end
 
+  # The maximum size (in +bytes+) to buffer in memory before
+  # resorting to a temporary file.  Default is 112 kilobytes.
+  # This option has no effect if "rewindable_input" is set to
+  # +false+.
+  def client_body_buffer_size(bytes)
+    set_int(:client_body_buffer_size, bytes, 0)
+  end
+
   # Allow redirecting $stderr to a given path.  Unlike doing this from
   # the shell, this allows the unicorn process to know the path its
   # writing to and rotate the file if it is used for logging.  The
@@ -470,6 +472,11 @@ class Unicorn::Configurator
   end
 
 private
+  def set_int(var, n, min)
+    Integer === n or raise ArgumentError, "not an integer: #{var}=#{n.inspect}"
+    n >= min or raise ArgumentError, "too low (< #{min}): #{var}=#{n.inspect}"
+    set[var] = n
+  end
 
   def set_path(var, path) #:nodoc:
     case path

data/lib/unicorn/const.rb

@@ -7,8 +7,8 @@
 # improve things much compared to constants.
 module Unicorn::Const
 
-  # The current version of Unicorn, currently 3.0.1
-  UNICORN_VERSION = "3.0.1"
+  # The current version of Unicorn, currently 3.1.0
+  UNICORN_VERSION = "3.1.0"
 
   # default TCP listen host address (0.0.0.0, all interfaces)
   DEFAULT_HOST = "0.0.0.0"
@@ -23,7 +23,8 @@ module Unicorn::Const
   CHUNK_SIZE = 16 * 1024
 
   # Maximum request body size before it is moved out of memory and into a
-  # temporary file for reading (112 kilobytes).
+  # temporary file for reading (112 kilobytes).  This is the default
+  # value of of client_body_buffer_size.
   MAX_BODY = 1024 * 112
 
   # :stopdoc:

data/lib/unicorn/http_server.rb

@@ -364,6 +364,14 @@ class Unicorn::HttpServer
                                 Unicorn::TeeInput : Unicorn::StreamInput
   end
 
+  def client_body_buffer_size
+    Unicorn::TeeInput.client_body_buffer_size
+  end
+
+  def client_body_buffer_size=(bytes)
+    Unicorn::TeeInput.client_body_buffer_size = bytes
+  end
+
   private
 
   # wait for a signal hander to wake us up and then consume the pipe

data/lib/unicorn/tee_input.rb

@@ -16,12 +16,24 @@ class Unicorn::TeeInput < Unicorn::StreamInput
   # resorting to a temporary file.  Default is 112 kilobytes.
   @@client_body_buffer_size = Unicorn::Const::MAX_BODY
 
+  # sets the maximum size of request bodies to buffer in memory,
+  # amounts larger than this are buffered to the filesystem
+  def self.client_body_buffer_size=(bytes)
+    @@client_body_buffer_size = bytes
+  end
+
+  # returns the maximum size of request bodies to buffer in memory,
+  # amounts larger than this are buffered to the filesystem
+  def self.client_body_buffer_size
+    @@client_body_buffer_size
+  end
+
   # Initializes a new TeeInput object.  You normally do not have to call
   # this unless you are writing an HTTP server.
   def initialize(socket, request)
     @len = request.content_length
     super
-    @tmp = @len && @len < @@client_body_buffer_size ?
+    @tmp = @len && @len <= @@client_body_buffer_size ?
            StringIO.new("") : Unicorn::TmpIO.new
   end
 
@@ -43,10 +55,10 @@ class Unicorn::TeeInput < Unicorn::StreamInput
   # specified +length+ in a loop until it returns +nil+.
   def size
     @len and return @len
-    pos = @bytes_read
+    pos = @tmp.pos
     consume!
     @tmp.pos = pos
-    @len = @bytes_read
+    @len = @tmp.size
   end
 
   # :call-seq:
@@ -92,7 +104,7 @@ class Unicorn::TeeInput < Unicorn::StreamInput
   # the offset (zero) of the +ios+ pointer.  Subsequent reads will
   # start from the beginning of the previously-buffered input.
   def rewind
-    return 0 if @bytes_read == 0
+    return 0 if 0 == @tmp.size
     consume! if @socket
     @tmp.rewind # Rack does not specify what the return value is here
   end

data/t/rack-input-tests.ru

@@ -0,0 +1,21 @@
+# SHA1 checksum generator
+require 'digest/sha1'
+use Rack::ContentLength
+cap = 16384
+app = lambda do |env|
+  /\A100-continue\z/i =~ env['HTTP_EXPECT'] and
+    return [ 100, {}, [] ]
+  digest = Digest::SHA1.new
+  input = env['rack.input']
+  input.size if env["PATH_INFO"] == "/size_first"
+  input.rewind if env["PATH_INFO"] == "/rewind_first"
+  if buf = input.read(rand(cap))
+    begin
+      raise "#{buf.size} > #{cap}" if buf.size > cap
+      digest.update(buf)
+    end while input.read(rand(cap), buf)
+  end
+
+  [ 200, {'Content-Type' => 'text/plain'}, [ digest.hexdigest << "\n" ] ]
+end
+run app

data/t/t0015-configurator-internals.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 4 "configurator internals tests (from FAQ)"
+
+t_begin "setup and start" && {
+	unicorn_setup
+	cat >> $unicorn_config <<EOF
+HttpRequest::DEFAULTS["rack.url_scheme"] = "https"
+Configurator::DEFAULTS[:logger].formatter = Logger::Formatter.new
+EOF
+	unicorn -D -c $unicorn_config env.ru
+	unicorn_wait_start
+}
+
+t_begin "single request" && {
+	curl -sSfv http://$listen/ | grep '"rack.url_scheme"=>"https"'
+}
+
+t_begin "killing succeeds" && {
+	kill $unicorn_pid
+}
+
+t_begin "no errors" && check_stderr
+
+t_done

data/t/t0100-rack-input-tests.sh

@@ -0,0 +1,124 @@
+#!/bin/sh
+. ./test-lib.sh
+test -r random_blob || die "random_blob required, run with 'make $0'"
+
+t_plan 10 "rack.input read tests"
+
+t_begin "setup and startup" && {
+	rtmpfiles curl_out curl_err
+	unicorn_setup
+	unicorn -D rack-input-tests.ru -c $unicorn_config
+        blob_sha1=$(rsha1 < random_blob)
+	blob_size=$(wc -c < random_blob)
+        t_info "blob_sha1=$blob_sha1"
+	unicorn_wait_start
+}
+
+t_begin "corked identity request" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf 'PUT / HTTP/1.0\r\n'
+		printf 'Content-Length: %d\r\n\r\n' $blob_size
+		cat random_blob
+		wait
+		echo ok > $ok
+	) | ( sleep 1 && socat - TCP4:$listen > $fifo )
+	test 1 -eq $(grep $blob_sha1 $tmp |wc -l)
+	test x"$(cat $ok)" = xok
+}
+
+t_begin "corked chunked request" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		content-md5-put < random_blob
+		wait
+		echo ok > $ok
+	) | ( sleep 1 && socat - TCP4:$listen > $fifo )
+	test 1 -eq $(grep $blob_sha1 $tmp |wc -l)
+	test x"$(cat $ok)" = xok
+}
+
+t_begin "corked identity request (input#size first)" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf 'PUT /size_first HTTP/1.0\r\n'
+		printf 'Content-Length: %d\r\n\r\n' $blob_size
+		cat random_blob
+		wait
+		echo ok > $ok
+	) | ( sleep 1 && socat - TCP4:$listen > $fifo )
+	test 1 -eq $(grep $blob_sha1 $tmp |wc -l)
+	test x"$(cat $ok)" = xok
+}
+
+t_begin "corked identity request (input#rewind first)" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf 'PUT /rewind_first HTTP/1.0\r\n'
+		printf 'Content-Length: %d\r\n\r\n' $blob_size
+		cat random_blob
+		wait
+		echo ok > $ok
+	) | ( sleep 1 && socat - TCP4:$listen > $fifo )
+	test 1 -eq $(grep $blob_sha1 $tmp |wc -l)
+	test x"$(cat $ok)" = xok
+}
+
+t_begin "corked chunked request (input#size first)" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf 'PUT /size_first HTTP/1.1\r\n'
+		printf 'Host: example.com\r\n'
+		printf 'Transfer-Encoding: chunked\r\n'
+		printf 'Trailer: Content-MD5\r\n'
+		printf '\r\n'
+		content-md5-put --no-headers < random_blob
+		wait
+		echo ok > $ok
+	) | ( sleep 1 && socat - TCP4:$listen > $fifo )
+	test 1 -eq $(grep $blob_sha1 $tmp |wc -l)
+	test 1 -eq $(grep $blob_sha1 $tmp |wc -l)
+	test x"$(cat $ok)" = xok
+}
+
+t_begin "corked chunked request (input#rewind first)" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf 'PUT /rewind_first HTTP/1.1\r\n'
+		printf 'Host: example.com\r\n'
+		printf 'Transfer-Encoding: chunked\r\n'
+		printf 'Trailer: Content-MD5\r\n'
+		printf '\r\n'
+		content-md5-put --no-headers < random_blob
+		wait
+		echo ok > $ok
+	) | ( sleep 1 && socat - TCP4:$listen > $fifo )
+	test 1 -eq $(grep $blob_sha1 $tmp |wc -l)
+	test x"$(cat $ok)" = xok
+}
+
+t_begin "regular request" && {
+	curl -sSf -T random_blob http://$listen/ > $curl_out 2> $curl_err
+        test x$blob_sha1 = x$(cat $curl_out)
+        test ! -s $curl_err
+}
+
+t_begin "chunked request" && {
+	curl -sSf -T- < random_blob http://$listen/ > $curl_out 2> $curl_err
+        test x$blob_sha1 = x$(cat $curl_out)
+        test ! -s $curl_err
+}
+
+dbgcat r_err
+
+t_begin "shutdown" && {
+	kill $unicorn_pid
+}
+
+t_done

data/t/t0116-client_body_buffer_size.sh

@@ -0,0 +1,80 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 12 "client_body_buffer_size settings"
+
+t_begin "setup and start" && {
+	unicorn_setup
+	rtmpfiles unicorn_config_tmp one_meg
+	dd if=/dev/zero bs=1M count=1 of=$one_meg
+	cat >> $unicorn_config <<EOF
+after_fork do |server, worker|
+  File.open("$fifo", "wb") { |fp| fp.syswrite "START" }
+end
+EOF
+	cat $unicorn_config > $unicorn_config_tmp
+	echo client_body_buffer_size 0 >> $unicorn_config
+	unicorn -D -c $unicorn_config t0116.ru
+	unicorn_wait_start
+	fs_class=Unicorn::TmpIO
+	mem_class=StringIO
+
+	test x"$(cat $fifo)" = xSTART
+}
+
+t_begin "class for a zero-byte file should be StringIO" && {
+	> $tmp
+	test xStringIO = x"$(curl -T $tmp -sSf http://$listen/input_class)"
+}
+
+t_begin "class for a 1 byte file should be filesystem-backed" && {
+	echo > $tmp
+	test x$fs_class = x"$(curl -T $tmp -sSf http://$listen/tmp_class)"
+}
+
+t_begin "reload with default client_body_buffer_size" && {
+	mv $unicorn_config_tmp $unicorn_config
+	kill -HUP $unicorn_pid
+	test x"$(cat $fifo)" = xSTART
+}
+
+t_begin "class for a 1 byte file should be memory-backed" && {
+	echo > $tmp
+	test x$mem_class = x"$(curl -T $tmp -sSf http://$listen/tmp_class)"
+}
+
+t_begin "class for a random blob file should be filesystem-backed" && {
+	resp="$(curl -T random_blob -sSf http://$listen/tmp_class)"
+	test x$fs_class = x"$resp"
+}
+
+t_begin "one megabyte file should be filesystem-backed" && {
+	resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)"
+	test x$fs_class = x"$resp"
+}
+
+t_begin "reload with a big client_body_buffer_size" && {
+	echo "client_body_buffer_size(1024 * 1024)" >> $unicorn_config
+	kill -HUP $unicorn_pid
+	test x"$(cat $fifo)" = xSTART
+}
+
+t_begin "one megabyte file should be memory-backed" && {
+	resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)"
+	test x$mem_class = x"$resp"
+}
+
+t_begin "one megabyte + 1 byte file should be filesystem-backed" && {
+	echo >> $one_meg
+	resp="$(curl -T $one_meg -sSf http://$listen/tmp_class)"
+	test x$fs_class = x"$resp"
+}
+
+t_begin "killing succeeds" && {
+	kill $unicorn_pid
+}
+
+t_begin "check stderr" && {
+	check_stderr
+}
+
+t_done

data/t/t0116.ru

@@ -0,0 +1,16 @@
+#\ -E none
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+app = lambda do |env|
+  input = env['rack.input']
+  case env["PATH_INFO"]
+  when "/tmp_class"
+    body = input.instance_variable_get(:@tmp).class.name
+  when "/input_class"
+    body = input.class.name
+  else
+    return [ 500, {}, [] ]
+  end
+  [ 200, {}, [ body ] ]
+end
+run app

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: unicorn
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 3
   prerelease: false
   segments: 
   - 3
-  - 0
   - 1
-  version: 3.0.1
+  - 0
+  version: 3.1.0
 platform: ruby
 authors: 
 - Unicorn hackers
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-03 00:00:00 +00:00
+date: 2010-12-09 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -198,6 +198,7 @@ files:
 - t/my-tap-lib.sh
 - t/pid.ru
 - t/preread_input.ru
+- t/rack-input-tests.ru
 - t/rails3-app/.gitignore
 - t/rails3-app/Gemfile
 - t/rails3-app/Rakefile
@@ -248,6 +249,10 @@ files:
 - t/t0013.ru
 - t/t0014-rewindable-input-true.sh
 - t/t0014.ru
+- t/t0015-configurator-internals.sh
+- t/t0100-rack-input-tests.sh
+- t/t0116-client_body_buffer_size.sh
+- t/t0116.ru
 - t/t0300-rails3-basic.sh
 - t/t0301-rails3-missing-config-ru.sh
 - t/t0302-rails3-alt-working_directory.sh