unicorn 0.2.3 → 0.4.1

data/test/benchmark/README

@@ -0,0 +1,55 @@
+= Performance
+
+Unicorn is pretty fast, and we want it to get faster.  Unicorn strives
+to get HTTP requests to your application and write HTTP responses back
+as quickly as possible.  Unicorn does not do any background processing
+while your app runs, so your app will get all the CPU time provided to
+it by your OS kernel.
+
+A gentle reminder: Unicorn is NOT for serving clients over slow network
+connections.  Use nginx (or something similar) to complement Unicorn if
+you have slow clients.
+
+== dd.ru
+
+This is a pure I/O benchmark.  In the context of Unicorn, this is the
+only one that matters.  It is a standard rackup-compatible .ru file and
+may be used with other Rack-compatible servers.
+
+  unicorn -E none dd.ru
+
+You can change the size and number of chunks in the response with
+the "bs" and "count" environment variables.   The following command
+will cause dd.ru to return 4 chunks of 16384 bytes each, leading to
+65536 byte response:
+
+  bs=16384 count=4 unicorn -E none dd.ru
+
+Or if you want to add logging (small performance impact):
+
+  unicorn -E deployment dd.ru
+
+Eric runs then runs clients on a LAN it in several different ways:
+
+  client@host1 -> unicorn@host1(tcp)
+  client@host2 -> unicorn@host1(tcp)
+  client@host3 -> nginx@host1 -> unicorn@host1(tcp)
+  client@host3 -> nginx@host1 -> unicorn@host1(unix)
+  client@host3 -> nginx@host2 -> unicorn@host1(tcp)
+
+The benchmark client is usually httperf.
+
+Another gentle reminder: performance with slow networks/clients
+is NOT our problem.  That is the job of nginx (or similar).
+
+== request.rb, response.rb, big_request.rb
+
+These are micro-benchmarks designed to test internal components
+of Unicorn.  It assumes the internal Unicorn API is mostly stable.
+
+== Contributors
+
+This directory is maintained independently in the "benchmark" branch
+based against v0.1.0.  Only changes to this directory (test/benchmarks)
+are committed to this branch although the master branch may merge this
+branch occassionaly.

data/test/benchmark/big_request.rb

@@ -0,0 +1,35 @@
+require 'benchmark'
+require 'tempfile'
+require 'unicorn'
+nr = ENV['nr'] ? ENV['nr'].to_i : 100
+bs = ENV['bs'] ? ENV['bs'].to_i : (1024 * 1024)
+count = ENV['count'] ? ENV['count'].to_i : 4
+length = bs * count
+slice = (' ' * bs).freeze
+
+big = Tempfile.new('')
+def big.unicorn_peeraddr; '127.0.0.1'; end
+big.syswrite(
+"PUT /hello/world/puturl?abcd=efg&hi#anchor HTTP/1.0\r\n" \
+"Host: localhost\r\n" \
+"Accept: */*\r\n" \
+"Content-Length: #{length}\r\n" \
+"User-Agent: test-user-agent 0.1.0 (Mozilla compatible) 5.0 asdfadfasda\r\n" \
+"\r\n")
+count.times { big.syswrite(slice) }
+big.sysseek(0)
+big.fsync
+
+include Unicorn
+request = HttpRequest.new(Logger.new($stderr))
+
+Benchmark.bmbm do |x|
+  x.report("big") do
+    for i in 1..nr
+      request.read(big)
+      request.reset
+      big.sysseek(0)
+    end
+  end
+end
+

data/test/benchmark/dd.ru

@@ -0,0 +1,18 @@
+# This benchmark is the simplest test of the I/O facilities in
+# unicorn.  It is meant to return a fixed-sized blob to test
+# the performance of things in Unicorn, _NOT_ the app.
+#
+# Adjusting this benchmark is done via the "bs" (byte size) and "count"
+# environment variables.  "count" designates the count of elements of
+# "bs" length in the Rack response body.  The defaults are bs=4096, count=1
+# to return one 4096-byte chunk.
+bs = ENV['bs'] ? ENV['bs'].to_i : 4096
+count = ENV['count'] ? ENV['count'].to_i : 1
+slice = (' ' * bs).freeze
+body = (1..count).map { slice }.freeze
+hdr = {
+  'Content-Length' => (bs * count).to_s.freeze,
+  'Content-Type' => 'text/plain'.freeze
+}.freeze
+response = [ 200, hdr, body ].freeze
+run(lambda { |env| response })

data/test/benchmark/request.rb

@@ -0,0 +1,47 @@
+require 'benchmark'
+require 'unicorn'
+nr = ENV['nr'] ? ENV['nr'].to_i : 100000
+
+class TestClient
+  def initialize(response)
+    @response = (response.join("\r\n") << "\r\n\r\n").freeze
+  end
+  def sysread(len, buf)
+    buf.replace(@response)
+  end
+
+  def unicorn_peeraddr
+    '127.0.0.1'
+  end
+end
+
+small = TestClient.new([
+  'GET / HTTP/1.0',
+  'Host: localhost',
+  'Accept: */*',
+  'User-Agent: test-user-agent 0.1.0'
+])
+
+medium = TestClient.new([
+  'GET /hello/world/geturl?abcd=efg&hi#anchor HTTP/1.0',
+  'Host: localhost',
+  'Accept: */*',
+  'User-Agent: test-user-agent 0.1.0 (Mozilla compatible) 5.0 asdfadfasda'
+])
+
+include Unicorn
+request = HttpRequest.new(Logger.new($stderr))
+Benchmark.bmbm do |x|
+  x.report("small") do
+    for i in 1..nr
+      request.read(small)
+      request.reset
+    end
+  end
+  x.report("medium") do
+    for i in 1..nr
+      request.read(medium)
+      request.reset
+    end
+  end
+end

data/test/benchmark/response.rb

@@ -0,0 +1,29 @@
+require 'benchmark'
+require 'unicorn'
+
+class NullWriter
+  def syswrite(buf); buf.size; end
+  def close; end
+end
+
+include Unicorn
+
+socket = NullWriter.new
+bs = ENV['bs'] ? ENV['bs'].to_i : 4096
+count = ENV['count'] ? ENV['count'].to_i : 1
+slice = (' ' * bs).freeze
+body = (1..count).map { slice }.freeze
+hdr = {
+  'Content-Length' => (bs * count).to_s.freeze,
+  'Content-Type' => 'text/plain'.freeze
+}.freeze
+response = [ 200, hdr, body ].freeze
+
+nr = ENV['nr'] ? ENV['nr'].to_i : 100000
+Benchmark.bmbm do |x|
+  x.report do
+    for i in 1..nr
+      HttpResponse.write(socket.dup, response)
+    end
+  end
+end

data/test/exec/test_exec.rb

@@ -1,43 +1,34 @@
 # Copyright (c) 2009 Eric Wong
-STDIN.sync = STDOUT.sync = STDERR.sync = true
 require 'test/test_helper'
-require 'pathname'
-require 'tempfile'
-require 'fileutils'
 
 do_test = true
-DEFAULT_TRIES = 1000
-DEFAULT_RES = 0.2
-
 $unicorn_bin = ENV['UNICORN_TEST_BIN'] || "unicorn"
 redirect_test_io do
   do_test = system($unicorn_bin, '-v')
 end
 
 unless do_test
-  STDERR.puts "#{$unicorn_bin} not found in PATH=#{ENV['PATH']}, " \
-              "skipping this test"
+  warn "#{$unicorn_bin} not found in PATH=#{ENV['PATH']}, " \
+       "skipping this test"
 end
 
-begin
-  require 'rack'
-rescue LoadError
-  STDERR.puts "Unable to load Rack, skipping this test"
+unless try_require('rack')
+  warn "Unable to load Rack, skipping this test"
   do_test = false
 end
 
 class ExecTest < Test::Unit::TestCase
-  trap('QUIT', 'IGNORE')
+  trap(:QUIT, 'IGNORE')
 
   HI = <<-EOS
 use Rack::ContentLength
-run proc { |env| [ 200, { 'Content-Type' => 'text/plain' }, "HI\\n" ] }
+run proc { |env| [ 200, { 'Content-Type' => 'text/plain' }, [ "HI\\n" ] ] }
   EOS
 
   HELLO = <<-EOS
 class Hello
   def call(env)
-    [ 200, { 'Content-Type' => 'text/plain' }, "HI\\n" ]
+    [ 200, { 'Content-Type' => 'text/plain' }, [ "HI\\n" ] ]
   end
 end
   EOS
@@ -47,7 +38,6 @@ end
   HEAVY_CFG = <<-EOS
 worker_processes 4
 timeout 30
-backlog 128
 logger Logger.new('#{COMMON_TMP.path}')
 before_fork do |server, worker_nr|
   server.logger.info "before_fork: worker=\#{worker_nr}"
@@ -113,7 +103,7 @@ end
     pid_file = "#{@tmpdir}/test.pid"
     old_file = "#{pid_file}.oldbin"
     ucfg = Tempfile.new('unicorn_test_config')
-    ucfg.syswrite("listeners %w(#{@addr}:#{@port})\n")
+    ucfg.syswrite("listen %(#@addr:#@port)\n")
     ucfg.syswrite("pid %(#{pid_file})\n")
     ucfg.syswrite("logger Logger.new(%(#{@tmpdir}/log))\n")
     pid = xfork do
@@ -126,14 +116,16 @@ end
 
     wait_for_file(pid_file)
     Process.waitpid(pid)
-    Process.kill('USR2', File.read(pid_file).to_i)
+    Process.kill(:USR2, File.read(pid_file).to_i)
     wait_for_file(old_file)
     wait_for_file(pid_file)
-    Process.kill('QUIT', File.read(old_file).to_i)
+    old_pid = File.read(old_file).to_i
+    Process.kill(:QUIT, old_pid)
+    wait_for_death(old_pid)
 
     ucfg.syswrite("timeout %(#{pid_file})\n") # introduce a bug
     current_pid = File.read(pid_file).to_i
-    Process.kill('USR2', current_pid)
+    Process.kill(:USR2, current_pid)
 
     # wait for pid_file to restore itself
     tries = DEFAULT_TRIES
@@ -156,9 +148,11 @@ end
     # fix the bug
     ucfg.sysseek(0)
     ucfg.truncate(0)
-    ucfg.syswrite("listeners %w(#{@addr}:#{@port} #{@addr}:#{port2})\n")
+    ucfg.syswrite("listen %(#@addr:#@port)\n")
+    ucfg.syswrite("listen %(#@addr:#{port2})\n")
     ucfg.syswrite("pid %(#{pid_file})\n")
-    Process.kill('USR2', current_pid)
+    assert_nothing_raised { Process.kill(:USR2, current_pid) }
+
     wait_for_file(old_file)
     wait_for_file(pid_file)
     new_pid = File.read(pid_file).to_i
@@ -170,8 +164,8 @@ end
     assert_equal String, results[1].class
 
     assert_nothing_raised do
-      Process.kill('QUIT', current_pid)
-      Process.kill('QUIT', new_pid)
+      Process.kill(:QUIT, current_pid)
+      Process.kill(:QUIT, new_pid)
     end
   end
 
@@ -192,14 +186,16 @@ end
 
     wait_for_file(pid_file)
     Process.waitpid(pid)
-    Process.kill('USR2', File.read(pid_file).to_i)
+    Process.kill(:USR2, File.read(pid_file).to_i)
     wait_for_file(old_file)
     wait_for_file(pid_file)
-    Process.kill('QUIT', File.read(old_file).to_i)
+    old_pid = File.read(old_file).to_i
+    Process.kill(:QUIT, old_pid)
+    wait_for_death(old_pid)
 
     File.unlink("config.ru") # break reloading
     current_pid = File.read(pid_file).to_i
-    Process.kill('USR2', current_pid)
+    Process.kill(:USR2, current_pid)
 
     # wait for pid_file to restore itself
     tries = DEFAULT_TRIES
@@ -210,17 +206,17 @@ end
     rescue Errno::ENOENT
       (sleep(DEFAULT_RES) and (tries -= 1) > 0) and retry
     end
-    assert_equal current_pid, File.read(pid_file).to_i
 
     tries = DEFAULT_TRIES
     while File.exist?(old_file)
       (sleep(DEFAULT_RES) and (tries -= 1) > 0) or break
     end
     assert ! File.exist?(old_file), "oldbin=#{old_file} gone"
+    assert_equal current_pid, File.read(pid_file).to_i
 
     # fix the bug
     File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
-    Process.kill('USR2', current_pid)
+    assert_nothing_raised { Process.kill(:USR2, current_pid) }
     wait_for_file(old_file)
     wait_for_file(pid_file)
     new_pid = File.read(pid_file).to_i
@@ -230,24 +226,21 @@ end
     assert_equal String, results[0].class
 
     assert_nothing_raised do
-      Process.kill('QUIT', current_pid)
-      Process.kill('QUIT', new_pid)
+      Process.kill(:QUIT, current_pid)
+      Process.kill(:QUIT, new_pid)
     end
   end
 
-  def test_unicorn_config_listeners_overrides_cli
-    port2 = unused_port(@addr)
+  def test_unicorn_config_listen_with_options
     File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
-    # listeners = [ ... ]  => should _override_ command-line options
     ucfg = Tempfile.new('unicorn_test_config')
-    ucfg.syswrite("listeners %w(#{@addr}:#{@port})\n")
+    ucfg.syswrite("listen '#{@addr}:#{@port}', :backlog => 512,\n")
+    ucfg.syswrite("                            :rcvbuf => 4096,\n")
+    ucfg.syswrite("                            :sndbuf => 4096\n")
     pid = xfork do
-      redirect_test_io do
-        exec($unicorn_bin, "-c#{ucfg.path}", "-l#{@addr}:#{port2}")
-      end
+      redirect_test_io { exec($unicorn_bin, "-c#{ucfg.path}") }
     end
     results = retry_hit(["http://#{@addr}:#{@port}/"])
-    assert_raises(Errno::ECONNREFUSED) { TCPSocket.new(@addr, port2) }
     assert_equal String, results[0].class
     assert_shutdown(pid)
   end
@@ -289,29 +282,30 @@ end
     rotate = Tempfile.new('unicorn_rotate')
     assert_nothing_raised do
       File.rename(COMMON_TMP.path, rotate.path)
-      Process.kill('USR1', pid)
+      Process.kill(:USR1, pid)
     end
     wait_for_file(COMMON_TMP.path)
     assert File.exist?(COMMON_TMP.path), "#{COMMON_TMP.path} exists"
     # USR1 should've been passed to all workers
     tries = DEFAULT_TRIES
     log = File.readlines(rotate.path)
-    while (tries -= 1) > 0 && log.grep(/rotating logs\.\.\./).size < 4
+    while (tries -= 1) > 0 &&
+          log.grep(/rotating logs\.\.\./).size < 5
       sleep DEFAULT_RES
       log = File.readlines(rotate.path)
     end
-    assert_equal 4, log.grep(/worker=\d+ rotating logs\.\.\./).size
+    assert_equal 5, log.grep(/rotating logs\.\.\./).size
     assert_equal 0, log.grep(/done rotating logs/).size
 
     tries = DEFAULT_TRIES
     log = File.readlines(COMMON_TMP.path)
-    while (tries -= 1) > 0 && log.grep(/done rotating logs/).size < 4
+    while (tries -= 1) > 0 && log.grep(/done rotating logs/).size < 5
       sleep DEFAULT_RES
       log = File.readlines(COMMON_TMP.path)
     end
-    assert_equal 4, log.grep(/worker=\d+ done rotating logs/).size
+    assert_equal 5, log.grep(/done rotating logs/).size
     assert_equal 0, log.grep(/rotating logs\.\.\./).size
-    assert_nothing_raised { Process.kill('QUIT', pid) }
+    assert_nothing_raised { Process.kill(:QUIT, pid) }
     status = nil
     assert_nothing_raised { pid, status = Process.waitpid2(pid) }
     assert status.success?, "exited successfully"
@@ -430,7 +424,7 @@ end
       ucfg.syswrite("pid \"#{pid_file}\"\n")
       ucfg.syswrite("logger Logger.new('#{new_log.path}')\n")
       ucfg.close
-      Process.kill('HUP', pid)
+      Process.kill(:HUP, pid)
     end
 
     wait_for_file(new_sock_path)
@@ -473,114 +467,4 @@ end
     reexec_usr2_quit_test(new_pid, pid_file)
   end
 
-  private
-
-    # sometimes the server may not come up right away
-    def retry_hit(uris = [])
-      tries = DEFAULT_TRIES
-      begin
-        hit(uris)
-      rescue Errno::ECONNREFUSED => err
-        if (tries -= 1) > 0
-          sleep DEFAULT_RES
-          retry
-        end
-        raise err
-      end
-    end
-
-    def assert_shutdown(pid)
-      wait_master_ready("#{@tmpdir}/test_stderr.#{pid}.log")
-      assert_nothing_raised { Process.kill('QUIT', pid) }
-      status = nil
-      assert_nothing_raised { pid, status = Process.waitpid2(pid) }
-      assert status.success?, "exited successfully"
-    end
-
-    def wait_workers_ready(path, nr_workers)
-      tries = DEFAULT_TRIES
-      lines = []
-      while (tries -= 1) > 0
-        begin
-          lines = File.readlines(path).grep(/worker=\d+ spawned/)
-          lines.size == nr_workers and return
-        rescue Errno::ENOENT
-        end
-        sleep DEFAULT_RES
-      end
-      raise "#{nr_workers} workers never became ready:" \
-            "\n\t#{lines.join("\n\t")}\n"
-    end
-
-    def wait_master_ready(master_log)
-      tries = DEFAULT_TRIES
-      while (tries -= 1) > 0
-        begin
-          File.readlines(master_log).grep(/master process ready/)[0] and return
-        rescue Errno::ENOENT
-        end
-        sleep DEFAULT_RES
-      end
-      raise "master process never became ready"
-    end
-
-    def reexec_usr2_quit_test(pid, pid_file)
-      assert File.exist?(pid_file), "pid file OK"
-      assert ! File.exist?("#{pid_file}.oldbin"), "oldbin pid file"
-      assert_nothing_raised { Process.kill('USR2', pid) }
-      assert_nothing_raised { retry_hit(["http://#{@addr}:#{@port}/"]) }
-      wait_for_file("#{pid_file}.oldbin")
-      wait_for_file(pid_file)
-
-      # kill old master process
-      assert_not_equal pid, File.read(pid_file).to_i
-      assert_equal pid, File.read("#{pid_file}.oldbin").to_i
-      assert_nothing_raised { Process.kill('QUIT', pid) }
-      assert_not_equal pid, File.read(pid_file).to_i
-      assert_nothing_raised { retry_hit(["http://#{@addr}:#{@port}/"]) }
-      wait_for_file(pid_file)
-      assert_nothing_raised { retry_hit(["http://#{@addr}:#{@port}/"]) }
-      assert_nothing_raised { Process.kill('QUIT', File.read(pid_file).to_i) }
-    end
-
-    def reexec_basic_test(pid, pid_file)
-      results = retry_hit(["http://#{@addr}:#{@port}/"])
-      assert_equal String, results[0].class
-      assert_nothing_raised { Process.kill(0, pid) }
-      master_log = "#{@tmpdir}/test_stderr.#{pid}.log"
-      wait_master_ready(master_log)
-      File.truncate(master_log, 0)
-      nr = 50
-      kill_point = 2
-      assert_nothing_raised do
-        nr.times do |i|
-          hit(["http://#{@addr}:#{@port}/#{i}"])
-          i == kill_point and Process.kill('HUP', pid)
-        end
-      end
-      wait_master_ready(master_log)
-      assert File.exist?(pid_file), "pid=#{pid_file} exists"
-      new_pid = File.read(pid_file).to_i
-      assert_not_equal pid, new_pid
-      assert_nothing_raised { Process.kill(0, new_pid) }
-      assert_nothing_raised { Process.kill('QUIT', new_pid) }
-    end
-
-    def wait_for_file(path)
-      tries = DEFAULT_TRIES
-      while (tries -= 1) > 0 && ! File.exist?(path)
-        sleep DEFAULT_RES
-      end
-      assert File.exist?(path), "path=#{path} exists #{caller.inspect}"
-    end
-
-    def xfork(&block)
-      fork do
-        ObjectSpace.each_object(Tempfile) do |tmp|
-          ObjectSpace.undefine_finalizer(tmp)
-        end
-        yield
-      end
-    end
-
 end if do_test