unicorn 5.0.1 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +5 -5
  2. data/.manifest +11 -5
  3. data/.olddoc.yml +16 -6
  4. data/Application_Timeouts +4 -4
  5. data/CONTRIBUTORS +6 -2
  6. data/Documentation/.gitignore +1 -3
  7. data/Documentation/unicorn.1 +222 -0
  8. data/Documentation/unicorn_rails.1 +207 -0
  9. data/FAQ +1 -1
  10. data/GIT-VERSION-FILE +1 -1
  11. data/GIT-VERSION-GEN +1 -1
  12. data/GNUmakefile +118 -58
  13. data/HACKING +2 -10
  14. data/ISSUES +40 -35
  15. data/KNOWN_ISSUES +2 -2
  16. data/LATEST +23 -28
  17. data/LICENSE +2 -2
  18. data/Links +13 -11
  19. data/NEWS +612 -0
  20. data/README +30 -29
  21. data/SIGNALS +1 -1
  22. data/Sandbox +8 -7
  23. data/TODO +0 -2
  24. data/TUNING +19 -1
  25. data/archive/slrnpull.conf +1 -1
  26. data/bin/unicorn +3 -1
  27. data/bin/unicorn_rails +2 -2
  28. data/examples/big_app_gc.rb +1 -1
  29. data/examples/init.sh +36 -8
  30. data/examples/logrotate.conf +17 -2
  31. data/examples/nginx.conf +4 -3
  32. data/examples/unicorn.conf.minimal.rb +2 -2
  33. data/examples/unicorn.conf.rb +2 -2
  34. data/examples/unicorn@.service +14 -0
  35. data/ext/unicorn_http/c_util.h +5 -13
  36. data/ext/unicorn_http/common_field_optimization.h +22 -5
  37. data/ext/unicorn_http/epollexclusive.h +124 -0
  38. data/ext/unicorn_http/ext_help.h +0 -44
  39. data/ext/unicorn_http/extconf.rb +32 -6
  40. data/ext/unicorn_http/global_variables.h +2 -2
  41. data/ext/unicorn_http/httpdate.c +2 -1
  42. data/ext/unicorn_http/unicorn_http.c +853 -498
  43. data/ext/unicorn_http/unicorn_http.rl +86 -30
  44. data/ext/unicorn_http/unicorn_http_common.rl +1 -1
  45. data/lib/unicorn/configurator.rb +93 -13
  46. data/lib/unicorn/http_request.rb +101 -11
  47. data/lib/unicorn/http_response.rb +8 -4
  48. data/lib/unicorn/http_server.rb +141 -72
  49. data/lib/unicorn/launcher.rb +1 -1
  50. data/lib/unicorn/oob_gc.rb +6 -6
  51. data/lib/unicorn/select_waiter.rb +6 -0
  52. data/lib/unicorn/socket_helper.rb +23 -7
  53. data/lib/unicorn/stream_input.rb +5 -4
  54. data/lib/unicorn/tee_input.rb +8 -10
  55. data/lib/unicorn/tmpio.rb +8 -2
  56. data/lib/unicorn/util.rb +3 -3
  57. data/lib/unicorn/version.rb +1 -1
  58. data/lib/unicorn/worker.rb +33 -8
  59. data/lib/unicorn.rb +55 -29
  60. data/man/man1/unicorn.1 +120 -118
  61. data/man/man1/unicorn_rails.1 +106 -107
  62. data/t/GNUmakefile +3 -72
  63. data/t/README +4 -4
  64. data/t/t0011-active-unix-socket.sh +1 -1
  65. data/t/t0012-reload-empty-config.sh +2 -1
  66. data/t/t0301-no-default-middleware-ignored-in-config.sh +25 -0
  67. data/t/t0301.ru +13 -0
  68. data/t/test-lib.sh +4 -3
  69. data/test/benchmark/README +14 -4
  70. data/test/benchmark/ddstream.ru +50 -0
  71. data/test/benchmark/readinput.ru +40 -0
  72. data/test/benchmark/uconnect.perl +66 -0
  73. data/test/exec/test_exec.rb +26 -24
  74. data/test/test_helper.rb +38 -30
  75. data/test/unit/test_ccc.rb +91 -0
  76. data/test/unit/test_droplet.rb +1 -1
  77. data/test/unit/test_http_parser.rb +46 -16
  78. data/test/unit/test_http_parser_ng.rb +81 -0
  79. data/test/unit/test_request.rb +10 -10
  80. data/test/unit/test_server.rb +86 -12
  81. data/test/unit/test_signals.rb +8 -8
  82. data/test/unit/test_socket_helper.rb +13 -9
  83. data/test/unit/test_upload.rb +9 -14
  84. data/test/unit/test_util.rb +31 -5
  85. data/test/unit/test_waiter.rb +34 -0
  86. data/unicorn.gemspec +21 -22
  87. metadata +21 -28
  88. data/Documentation/GNUmakefile +0 -30
  89. data/Documentation/unicorn.1.txt +0 -188
  90. data/Documentation/unicorn_rails.1.txt +0 -175
  91. data/t/hijack.ru +0 -43
  92. data/t/t0200-rack-hijack.sh +0 -30
@@ -0,0 +1,66 @@
1
+ #!/usr/bin/perl -w
2
+ # Benchmark script to spawn some processes and hammer a local unicorn
3
+ # to test accept loop performance. This only does Unix sockets.
4
+ # There's plenty of TCP benchmarking tools out there, and TCP port reuse
5
+ # has predictability problems since unicorn can't do persistent connections.
6
+ # Written in Perl for the same reason: predictability.
7
+ # Ruby GC is not as predictable as Perl refcounting.
8
+ use strict;
9
+ use Socket qw(AF_UNIX SOCK_STREAM sockaddr_un);
10
+ use POSIX qw(:sys_wait_h);
11
+ use Getopt::Std;
12
+ # -c / -n switches stolen from ab(1)
13
+ my $usage = "$0 [-c CONCURRENCY] [-n NUM_REQUESTS] SOCKET_PATH\n";
14
+ our $opt_c = 2;
15
+ our $opt_n = 1000;
16
+ getopts('c:n:') or die $usage;
17
+ my $unix_path = shift or die $usage;
18
+ use constant REQ => "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
19
+ use constant REQ_LEN => length(REQ);
20
+ use constant BUFSIZ => 8192;
21
+ $^F = 99; # don't waste syscall time with FD_CLOEXEC
22
+
23
+ my %workers; # pid => worker num
24
+ die "-n $opt_n not evenly divisible by -c $opt_c\n" if $opt_n % $opt_c;
25
+ my $n_per_worker = $opt_n / $opt_c;
26
+ my $addr = sockaddr_un($unix_path);
27
+
28
+ for my $num (1..$opt_c) {
29
+ defined(my $pid = fork) or die "fork failed: $!\n";
30
+ if ($pid) {
31
+ $workers{$pid} = $num;
32
+ } else {
33
+ work($n_per_worker);
34
+ }
35
+ }
36
+
37
+ reap_worker(0) while scalar keys %workers;
38
+ exit;
39
+
40
+ sub work {
41
+ my ($n) = @_;
42
+ my ($buf, $x);
43
+ for (1..$n) {
44
+ socket(S, AF_UNIX, SOCK_STREAM, 0) or die "socket: $!";
45
+ connect(S, $addr) or die "connect: $!";
46
+ defined($x = syswrite(S, REQ)) or die "write: $!";
47
+ $x == REQ_LEN or die "short write: $x != ".REQ_LEN."\n";
48
+ do {
49
+ $x = sysread(S, $buf, BUFSIZ);
50
+ unless (defined $x) {
51
+ next if $!{EINTR};
52
+ die "sysread: $!\n";
53
+ }
54
+ } until ($x == 0);
55
+ }
56
+ exit 0;
57
+ }
58
+
59
+ sub reap_worker {
60
+ my ($flags) = @_;
61
+ my $pid = waitpid(-1, $flags);
62
+ return if !defined $pid || $pid <= 0;
63
+ my $p = delete $workers{$pid} || '(unknown)';
64
+ warn("$pid [$p] exited with $?\n") if $?;
65
+ $p;
66
+ }
@@ -45,8 +45,9 @@ end
45
45
 
46
46
  COMMON_TMP = Tempfile.new('unicorn_tmp') unless defined?(COMMON_TMP)
47
47
 
48
+ HEAVY_WORKERS = 2
48
49
  HEAVY_CFG = <<-EOS
49
- worker_processes 4
50
+ worker_processes #{HEAVY_WORKERS}
50
51
  timeout 30
51
52
  logger Logger.new('#{COMMON_TMP.path}')
52
53
  before_fork do |server, worker|
@@ -97,6 +98,9 @@ run lambda { |env|
97
98
  end
98
99
 
99
100
  def test_sd_listen_fds_emulation
101
+ # [ruby-core:69895] [Bug #11336] fixed by r51576
102
+ return if RUBY_VERSION.to_f < 2.3
103
+
100
104
  File.open("config.ru", "wb") { |fp| fp.write(HI) }
101
105
  sock = TCPServer.new(@addr, @port)
102
106
 
@@ -119,14 +123,12 @@ run lambda { |env|
119
123
  res = hit(["http://#@addr:#@port/"])
120
124
  assert_equal [ "HI\n" ], res
121
125
  assert_shutdown(pid)
122
- assert_equal 1, sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).int,
126
+ assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool,
123
127
  'unicorn should always set SO_KEEPALIVE on inherited sockets'
124
128
  end
125
129
  ensure
126
130
  sock.close if sock
127
- # disabled test on old Rubies: https://bugs.ruby-lang.org/issues/11336
128
- # [ruby-core:69895] [Bug #11336] fixed by r51576
129
- end if RUBY_VERSION.to_f >= 2.3
131
+ end
130
132
 
131
133
  def test_inherit_listener_unspecified
132
134
  File.open("config.ru", "wb") { |fp| fp.write(HI) }
@@ -142,7 +144,7 @@ run lambda { |env|
142
144
  res = hit(["http://#@addr:#@port/"])
143
145
  assert_equal [ "HI\n" ], res
144
146
  assert_shutdown(pid)
145
- assert_equal 1, sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).int,
147
+ assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool,
146
148
  'unicorn should always set SO_KEEPALIVE on inherited sockets'
147
149
  ensure
148
150
  sock.close if sock
@@ -192,8 +194,8 @@ EOF
192
194
  assert_equal other.path, results.first
193
195
 
194
196
  Process.kill(:QUIT, pid)
195
- ensure
196
- FileUtils.rmtree(other.path)
197
+ ensure
198
+ FileUtils.rmtree(other.path)
197
199
  end
198
200
 
199
201
  def test_working_directory
@@ -228,8 +230,8 @@ EOF
228
230
  assert_equal other.path, results.first
229
231
 
230
232
  Process.kill(:QUIT, pid)
231
- ensure
232
- FileUtils.rmtree(other.path)
233
+ ensure
234
+ FileUtils.rmtree(other.path)
233
235
  end
234
236
 
235
237
  def test_working_directory_controls_relative_paths
@@ -270,11 +272,10 @@ EOF
270
272
  wait_master_ready("#{other.path}/stderr_log_here")
271
273
 
272
274
  Process.kill(:QUIT, pid)
273
- ensure
274
- FileUtils.rmtree(other.path)
275
+ ensure
276
+ FileUtils.rmtree(other.path)
275
277
  end
276
278
 
277
-
278
279
  def test_exit_signals
279
280
  %w(INT TERM QUIT).each do |sig|
280
281
  File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
@@ -573,7 +574,7 @@ EOF
573
574
  assert_equal String, results[0].class
574
575
  worker_pid = results[0].to_i
575
576
  assert_not_equal pid, worker_pid
576
- s = UNIXSocket.new(tmp.path)
577
+ s = unix_socket(tmp.path)
577
578
  s.syswrite("GET / HTTP/1.0\r\n\r\n")
578
579
  results = ''
579
580
  loop { results << s.sysread(4096) } rescue nil
@@ -606,6 +607,7 @@ EOF
606
607
  def test_weird_config_settings
607
608
  File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
608
609
  ucfg = Tempfile.new('unicorn_test_config')
610
+ proc_total = HEAVY_WORKERS + 1 # + 1 for master
609
611
  ucfg.syswrite(HEAVY_CFG)
610
612
  pid = xfork do
611
613
  redirect_test_io do
@@ -616,9 +618,9 @@ EOF
616
618
  results = retry_hit(["http://#{@addr}:#{@port}/"])
617
619
  assert_equal String, results[0].class
618
620
  wait_master_ready(COMMON_TMP.path)
619
- wait_workers_ready(COMMON_TMP.path, 4)
621
+ wait_workers_ready(COMMON_TMP.path, HEAVY_WORKERS)
620
622
  bf = File.readlines(COMMON_TMP.path).grep(/\bbefore_fork: worker=/)
621
- assert_equal 4, bf.size
623
+ assert_equal HEAVY_WORKERS, bf.size
622
624
  rotate = Tempfile.new('unicorn_rotate')
623
625
 
624
626
  File.rename(COMMON_TMP.path, rotate.path)
@@ -630,20 +632,20 @@ EOF
630
632
  tries = DEFAULT_TRIES
631
633
  log = File.readlines(rotate.path)
632
634
  while (tries -= 1) > 0 &&
633
- log.grep(/reopening logs\.\.\./).size < 5
635
+ log.grep(/reopening logs\.\.\./).size < proc_total
634
636
  sleep DEFAULT_RES
635
637
  log = File.readlines(rotate.path)
636
638
  end
637
- assert_equal 5, log.grep(/reopening logs\.\.\./).size
639
+ assert_equal proc_total, log.grep(/reopening logs\.\.\./).size
638
640
  assert_equal 0, log.grep(/done reopening logs/).size
639
641
 
640
642
  tries = DEFAULT_TRIES
641
643
  log = File.readlines(COMMON_TMP.path)
642
- while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < 5
644
+ while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < proc_total
643
645
  sleep DEFAULT_RES
644
646
  log = File.readlines(COMMON_TMP.path)
645
647
  end
646
- assert_equal 5, log.grep(/done reopening logs/).size
648
+ assert_equal proc_total, log.grep(/done reopening logs/).size
647
649
  assert_equal 0, log.grep(/reopening logs\.\.\./).size
648
650
 
649
651
  Process.kill(:QUIT, pid)
@@ -730,7 +732,7 @@ EOF
730
732
  wait_for_file(sock_path)
731
733
  assert File.socket?(sock_path)
732
734
 
733
- sock = UNIXSocket.new(sock_path)
735
+ sock = unix_socket(sock_path)
734
736
  sock.syswrite("GET / HTTP/1.0\r\n\r\n")
735
737
  results = sock.sysread(4096)
736
738
 
@@ -740,7 +742,7 @@ EOF
740
742
  wait_for_file(sock_path)
741
743
  assert File.socket?(sock_path)
742
744
 
743
- sock = UNIXSocket.new(sock_path)
745
+ sock = unix_socket(sock_path)
744
746
  sock.syswrite("GET / HTTP/1.0\r\n\r\n")
745
747
  results = sock.sysread(4096)
746
748
 
@@ -775,7 +777,7 @@ EOF
775
777
  assert_equal pid, File.read(pid_file).to_i
776
778
  assert File.socket?(sock_path), "socket created"
777
779
 
778
- sock = UNIXSocket.new(sock_path)
780
+ sock = unix_socket(sock_path)
779
781
  sock.syswrite("GET / HTTP/1.0\r\n\r\n")
780
782
  results = sock.sysread(4096)
781
783
 
@@ -801,7 +803,7 @@ EOF
801
803
  wait_for_file(new_sock_path)
802
804
  assert File.socket?(new_sock_path), "socket exists"
803
805
  @sockets.each do |path|
804
- sock = UNIXSocket.new(path)
806
+ sock = unix_socket(path)
805
807
  sock.syswrite("GET / HTTP/1.0\r\n\r\n")
806
808
  results = sock.sysread(4096)
807
809
  assert_equal String, results.class
data/test/test_helper.rb CHANGED
@@ -28,22 +28,43 @@ require 'tempfile'
28
28
  require 'fileutils'
29
29
  require 'logger'
30
30
  require 'unicorn'
31
+ require 'io/nonblock'
31
32
 
32
33
  if ENV['DEBUG']
33
34
  require 'ruby-debug'
34
35
  Debugger.start
35
36
  end
36
37
 
38
+ unless RUBY_VERSION < '3.1'
39
+ warn "Unicorn was only tested against MRI up to 3.0.\n" \
40
+ "It might not properly work with #{RUBY_VERSION}"
41
+ end
42
+
37
43
  def redirect_test_io
38
44
  orig_err = STDERR.dup
39
45
  orig_out = STDOUT.dup
40
- STDERR.reopen("test_stderr.#{$$}.log", "a")
41
- STDOUT.reopen("test_stdout.#{$$}.log", "a")
46
+ rdr_pid = $$
47
+ new_out = File.open("test_stdout.#$$.log", "a")
48
+ new_err = File.open("test_stderr.#$$.log", "a")
49
+ new_out.sync = new_err.sync = true
50
+
51
+ if tail = ENV['TAIL'] # "tail -F" if GNU, "tail -f" otherwise
52
+ require 'shellwords'
53
+ cmd = tail.shellsplit
54
+ cmd << new_out.path
55
+ cmd << new_err.path
56
+ pid = Process.spawn(*cmd, { 1 => 2, :pgroup => true })
57
+ sleep 0.1 # wait for tail(1) to startup
58
+ end
59
+ STDERR.reopen(new_err)
60
+ STDOUT.reopen(new_out)
42
61
  STDERR.sync = STDOUT.sync = true
43
62
 
44
63
  at_exit do
45
- File.unlink("test_stderr.#{$$}.log") rescue nil
46
- File.unlink("test_stdout.#{$$}.log") rescue nil
64
+ if rdr_pid == $$
65
+ File.unlink(new_out.path) rescue nil
66
+ File.unlink(new_err.path) rescue nil
67
+ end
47
68
  end
48
69
 
49
70
  begin
@@ -51,6 +72,7 @@ def redirect_test_io
51
72
  ensure
52
73
  STDERR.reopen(orig_err)
53
74
  STDOUT.reopen(orig_out)
75
+ Process.kill(:TERM, pid) if pid
54
76
  end
55
77
  end
56
78
 
@@ -265,34 +287,20 @@ def wait_for_death(pid)
265
287
  raise "PID:#{pid} never died!"
266
288
  end
267
289
 
268
- # executes +cmd+ and chunks its STDOUT
269
- def chunked_spawn(stdout, *cmd)
270
- fork {
271
- crd, cwr = IO.pipe
272
- crd.binmode
273
- cwr.binmode
274
- crd.sync = cwr.sync = true
275
-
276
- pid = fork {
277
- STDOUT.reopen(cwr)
278
- crd.close
279
- cwr.close
280
- exec(*cmd)
281
- }
282
- cwr.close
283
- begin
284
- buf = crd.readpartial(16384)
285
- stdout.write("#{'%x' % buf.size}\r\n#{buf}")
286
- rescue EOFError
287
- stdout.write("0\r\n")
288
- pid, status = Process.waitpid(pid)
289
- exit status.exitstatus
290
- end while true
291
- }
292
- end
293
-
294
290
  def reset_sig_handlers
295
291
  %w(WINCH QUIT INT TERM USR1 USR2 HUP TTIN TTOU CHLD).each do |sig|
296
292
  trap(sig, "DEFAULT")
297
293
  end
298
294
  end
295
+
296
+ def tcp_socket(*args)
297
+ sock = TCPSocket.new(*args)
298
+ sock.nonblock = false
299
+ sock
300
+ end
301
+
302
+ def unix_socket(*args)
303
+ sock = UNIXSocket.new(*args)
304
+ sock.nonblock = false
305
+ sock
306
+ end
@@ -0,0 +1,91 @@
1
+ require 'socket'
2
+ require 'unicorn'
3
+ require 'io/wait'
4
+ require 'tempfile'
5
+ require 'test/unit'
6
+ require './test/test_helper'
7
+
8
+ class TestCccTCPI < Test::Unit::TestCase
9
+ def test_ccc_tcpi
10
+ start_pid = $$
11
+ host = '127.0.0.1'
12
+ srv = TCPServer.new(host, 0)
13
+ port = srv.addr[1]
14
+ err = Tempfile.new('unicorn_ccc')
15
+ rd, wr = IO.pipe
16
+ sleep_pipe = IO.pipe
17
+ pid = fork do
18
+ sleep_pipe[1].close
19
+ reqs = 0
20
+ rd.close
21
+ worker_pid = nil
22
+ app = lambda do |env|
23
+ worker_pid ||= begin
24
+ at_exit { wr.write(reqs.to_s) if worker_pid == $$ }
25
+ $$
26
+ end
27
+ reqs += 1
28
+
29
+ # will wake up when writer closes
30
+ sleep_pipe[0].read if env['PATH_INFO'] == '/sleep'
31
+
32
+ [ 200, [ %w(Content-Length 0), %w(Content-Type text/plain) ], [] ]
33
+ end
34
+ ENV['UNICORN_FD'] = srv.fileno.to_s
35
+ opts = {
36
+ listeners: [ "#{host}:#{port}" ],
37
+ stderr_path: err.path,
38
+ check_client_connection: true,
39
+ }
40
+ uni = Unicorn::HttpServer.new(app, opts)
41
+ uni.start.join
42
+ end
43
+ wr.close
44
+
45
+ # make sure the server is running, at least
46
+ client = tcp_socket(host, port)
47
+ client.write("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
48
+ assert client.wait(10), 'never got response from server'
49
+ res = client.read
50
+ assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first response'
51
+ assert_match %r{\r\n\r\n\z}, res, 'got end of response, server is ready'
52
+ client.close
53
+
54
+ # start a slow request...
55
+ sleeper = tcp_socket(host, port)
56
+ sleeper.write("GET /sleep HTTP/1.1\r\nHost: example.com\r\n\r\n")
57
+
58
+ # and a bunch of aborted ones
59
+ nr = 100
60
+ nr.times do |i|
61
+ client = tcp_socket(host, port)
62
+ client.write("GET /collections/#{rand(10000)} HTTP/1.1\r\n" \
63
+ "Host: example.com\r\n\r\n")
64
+ client.close
65
+ end
66
+ sleep_pipe[1].close # wake up the reader in the worker
67
+ res = sleeper.read
68
+ assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first sleeper response'
69
+ assert_match %r{\r\n\r\n\z}, res, 'got end of sleeper response'
70
+ sleeper.close
71
+ kpid = pid
72
+ pid = nil
73
+ Process.kill(:QUIT, kpid)
74
+ _, status = Process.waitpid2(kpid)
75
+ assert status.success?
76
+ reqs = rd.read.to_i
77
+ warn "server got #{reqs} requests with #{nr} CCC aborted\n" if $DEBUG
78
+ assert_operator reqs, :<, nr
79
+ assert_operator reqs, :>=, 2, 'first 2 requests got through, at least'
80
+ ensure
81
+ return if start_pid != $$
82
+ srv.close if srv
83
+ if pid
84
+ Process.kill(:QUIT, pid)
85
+ _, status = Process.waitpid2(pid)
86
+ assert status.success?
87
+ end
88
+ err.close! if err
89
+ rd.close if rd
90
+ end
91
+ end
@@ -4,7 +4,7 @@ require 'unicorn'
4
4
  class TestDroplet < Test::Unit::TestCase
5
5
  def test_create_many_droplets
6
6
  now = Time.now.to_i
7
- tmp = (0..1024).map do |i|
7
+ (0..1024).each do |i|
8
8
  droplet = Unicorn::Worker.new(i)
9
9
  assert droplet.respond_to?(:tick)
10
10
  assert_equal 0, droplet.tick
@@ -230,6 +230,24 @@ class HttpParserTest < Test::Unit::TestCase
230
230
  assert_equal expect, req['HTTP_X_SSL_BULLSHIT']
231
231
  end
232
232
 
233
+ def test_multiline_header_0d0a
234
+ parser = HttpParser.new
235
+ parser.buf << "GET / HTTP/1.0\r\n" \
236
+ "X-Multiline-Header: foo bar\r\n\tcha cha\r\n\tzha zha\r\n\r\n"
237
+ req = parser.env
238
+ assert_equal req, parser.parse
239
+ assert_equal 'foo bar cha cha zha zha', req['HTTP_X_MULTILINE_HEADER']
240
+ end
241
+
242
+ def test_multiline_header_0a
243
+ parser = HttpParser.new
244
+ parser.buf << "GET / HTTP/1.0\n" \
245
+ "X-Multiline-Header: foo bar\n\tcha cha\n\tzha zha\n\n"
246
+ req = parser.env
247
+ assert_equal req, parser.parse
248
+ assert_equal 'foo bar cha cha zha zha', req['HTTP_X_MULTILINE_HEADER']
249
+ end
250
+
233
251
  def test_continuation_eats_leading_spaces
234
252
  parser = HttpParser.new
235
253
  header = "GET / HTTP/1.1\r\n" \
@@ -833,22 +851,34 @@ class HttpParserTest < Test::Unit::TestCase
833
851
  assert_equal '', parser.env['HTTP_HOST']
834
852
  end
835
853
 
836
- # so we don't care about the portability of this test
837
- # if it doesn't leak on Linux, it won't leak anywhere else
838
- # unless your C compiler or platform is otherwise broken
839
- LINUX_PROC_PID_STATUS = "/proc/self/status"
840
- def test_memory_leak
841
- match_rss = /^VmRSS:\s+(\d+)/
842
- if File.read(LINUX_PROC_PID_STATUS) =~ match_rss
843
- before = $1.to_i
844
- 1000000.times { Unicorn::HttpParser.new }
845
- File.read(LINUX_PROC_PID_STATUS) =~ match_rss
846
- after = $1.to_i
847
- diff = after - before
848
- assert(diff < 10000, "memory grew more than 10M: #{diff}")
854
+ def test_memsize
855
+ require 'objspace'
856
+ if ObjectSpace.respond_to?(:memsize_of)
857
+ n = ObjectSpace.memsize_of(Unicorn::HttpParser.new)
858
+ assert_kind_of Integer, n
859
+ # need to update this when 128-bit machines come out
860
+ # n.b. actual struct size on 64-bit is 56 bytes + 40 bytes for RVALUE
861
+ # Ruby <= 2.2 objspace did not count the 40-byte RVALUE, 2.3 does.
862
+ assert_operator n, :<=, 96
863
+ assert_operator n, :>, 0
849
864
  end
850
- end if RUBY_PLATFORM =~ /linux/ &&
851
- File.readable?(LINUX_PROC_PID_STATUS) &&
852
- !defined?(RUBY_ENGINE)
865
+ rescue LoadError
866
+ # not all Ruby implementations have objspace
867
+ end
853
868
 
869
+ def test_dedupe
870
+ parser = HttpParser.new
871
+ # n.b. String#freeze optimization doesn't work under modern test-unit
872
+ exp = -'HTTP_HOST'
873
+ get = "GET / HTTP/1.1\r\nHost: example.com\r\nHavpbea-fhpxf: true\r\n\r\n"
874
+ assert parser.add_parse(get)
875
+ key = parser.env.keys.detect { |k| k == exp }
876
+ assert_same exp, key
877
+
878
+ if RUBY_VERSION.to_r >= 2.6 # 2.6.0-rc1+
879
+ exp = -'HTTP_HAVPBEA_FHPXF'
880
+ key = parser.env.keys.detect { |k| k == exp }
881
+ assert_same exp, key
882
+ end
883
+ end if RUBY_VERSION.to_r >= 2.5 && RUBY_ENGINE == 'ruby'
854
884
  end
@@ -11,6 +11,20 @@ class HttpParserNgTest < Test::Unit::TestCase
11
11
  @parser = HttpParser.new
12
12
  end
13
13
 
14
+ # RFC 7230 allows gzip/deflate/compress Transfer-Encoding,
15
+ # but "chunked" must be last if used
16
+ def test_is_chunked
17
+ [ 'chunked,chunked', 'chunked,gzip', 'chunked,gzip,chunked' ].each do |x|
18
+ assert_raise(HttpParserError) { HttpParser.is_chunked?(x) }
19
+ end
20
+ [ 'gzip, chunked', 'gzip,chunked', 'gzip ,chunked' ].each do |x|
21
+ assert HttpParser.is_chunked?(x)
22
+ end
23
+ [ 'gzip', 'xhunked', 'xchunked' ].each do |x|
24
+ assert !HttpParser.is_chunked?(x)
25
+ end
26
+ end
27
+
14
28
  def test_parser_max_len
15
29
  assert_raises(RangeError) do
16
30
  HttpParser.max_header_len = 0xffffffff + 1
@@ -566,6 +580,73 @@ class HttpParserNgTest < Test::Unit::TestCase
566
580
  end
567
581
  end
568
582
 
583
+ def test_duplicate_content_length
584
+ str = "PUT / HTTP/1.1\r\n" \
585
+ "Content-Length: 1\r\n" \
586
+ "Content-Length: 9\r\n" \
587
+ "\r\n"
588
+ assert_raises(HttpParserError) { @parser.headers({}, str) }
589
+ end
590
+
591
+ def test_chunked_overrides_content_length
592
+ order = [ 'Transfer-Encoding: chunked', 'Content-Length: 666' ]
593
+ %w(a b).each do |x|
594
+ str = "PUT /#{x} HTTP/1.1\r\n" \
595
+ "#{order.join("\r\n")}" \
596
+ "\r\n\r\na\r\nhelloworld\r\n0\r\n\r\n"
597
+ order.reverse!
598
+ env = @parser.headers({}, str)
599
+ assert_nil @parser.content_length
600
+ assert_equal 'chunked', env['HTTP_TRANSFER_ENCODING']
601
+ assert_equal '666', env['CONTENT_LENGTH'],
602
+ 'Content-Length logged so the app can log a possible client bug/attack'
603
+ @parser.filter_body(dst = '', str)
604
+ assert_equal 'helloworld', dst
605
+ @parser.parse # handle the non-existent trailer
606
+ assert @parser.next?
607
+ end
608
+ end
609
+
610
+ def test_chunked_order_good
611
+ str = "PUT /x HTTP/1.1\r\n" \
612
+ "Transfer-Encoding: gzip\r\n" \
613
+ "Transfer-Encoding: chunked\r\n" \
614
+ "\r\n"
615
+ env = @parser.headers({}, str)
616
+ assert_equal 'gzip,chunked', env['HTTP_TRANSFER_ENCODING']
617
+ assert_nil @parser.content_length
618
+
619
+ @parser.clear
620
+ str = "PUT /x HTTP/1.1\r\n" \
621
+ "Transfer-Encoding: gzip, chunked\r\n" \
622
+ "\r\n"
623
+ env = @parser.headers({}, str)
624
+ assert_equal 'gzip, chunked', env['HTTP_TRANSFER_ENCODING']
625
+ assert_nil @parser.content_length
626
+ end
627
+
628
+ def test_chunked_order_bad
629
+ str = "PUT /x HTTP/1.1\r\n" \
630
+ "Transfer-Encoding: chunked\r\n" \
631
+ "Transfer-Encoding: gzip\r\n" \
632
+ "\r\n"
633
+ assert_raise(HttpParserError) { @parser.headers({}, str) }
634
+ end
635
+
636
+ def test_double_chunked
637
+ str = "PUT /x HTTP/1.1\r\n" \
638
+ "Transfer-Encoding: chunked\r\n" \
639
+ "Transfer-Encoding: chunked\r\n" \
640
+ "\r\n"
641
+ assert_raise(HttpParserError) { @parser.headers({}, str) }
642
+
643
+ @parser.clear
644
+ str = "PUT /x HTTP/1.1\r\n" \
645
+ "Transfer-Encoding: chunked,chunked\r\n" \
646
+ "\r\n"
647
+ assert_raise(HttpParserError) { @parser.headers({}, str) }
648
+ end
649
+
569
650
  def test_backtrace_is_empty
570
651
  begin
571
652
  @parser.headers({}, "AAADFSFDSFD\r\n\r\n")
@@ -34,7 +34,7 @@ class RequestTest < Test::Unit::TestCase
34
34
  assert_equal '', env['REQUEST_PATH']
35
35
  assert_equal '', env['PATH_INFO']
36
36
  assert_equal '*', env['REQUEST_URI']
37
- res = @lint.call(env)
37
+ assert_kind_of Array, @lint.call(env)
38
38
  end
39
39
 
40
40
  def test_absolute_uri_with_query
@@ -44,7 +44,7 @@ class RequestTest < Test::Unit::TestCase
44
44
  assert_equal '/x', env['REQUEST_PATH']
45
45
  assert_equal '/x', env['PATH_INFO']
46
46
  assert_equal 'y=z', env['QUERY_STRING']
47
- res = @lint.call(env)
47
+ assert_kind_of Array, @lint.call(env)
48
48
  end
49
49
 
50
50
  def test_absolute_uri_with_fragment
@@ -55,7 +55,7 @@ class RequestTest < Test::Unit::TestCase
55
55
  assert_equal '/x', env['PATH_INFO']
56
56
  assert_equal '', env['QUERY_STRING']
57
57
  assert_equal 'frag', env['FRAGMENT']
58
- res = @lint.call(env)
58
+ assert_kind_of Array, @lint.call(env)
59
59
  end
60
60
 
61
61
  def test_absolute_uri_with_query_and_fragment
@@ -66,7 +66,7 @@ class RequestTest < Test::Unit::TestCase
66
66
  assert_equal '/x', env['PATH_INFO']
67
67
  assert_equal 'a=b', env['QUERY_STRING']
68
68
  assert_equal 'frag', env['FRAGMENT']
69
- res = @lint.call(env)
69
+ assert_kind_of Array, @lint.call(env)
70
70
  end
71
71
 
72
72
  def test_absolute_uri_unsupported_schemes
@@ -83,7 +83,7 @@ class RequestTest < Test::Unit::TestCase
83
83
  "Host: foo\r\n\r\n")
84
84
  env = @request.read(client)
85
85
  assert_equal "https", env['rack.url_scheme']
86
- res = @lint.call(env)
86
+ assert_kind_of Array, @lint.call(env)
87
87
  end
88
88
 
89
89
  def test_x_forwarded_proto_http
@@ -92,7 +92,7 @@ class RequestTest < Test::Unit::TestCase
92
92
  "Host: foo\r\n\r\n")
93
93
  env = @request.read(client)
94
94
  assert_equal "http", env['rack.url_scheme']
95
- res = @lint.call(env)
95
+ assert_kind_of Array, @lint.call(env)
96
96
  end
97
97
 
98
98
  def test_x_forwarded_proto_invalid
@@ -101,7 +101,7 @@ class RequestTest < Test::Unit::TestCase
101
101
  "Host: foo\r\n\r\n")
102
102
  env = @request.read(client)
103
103
  assert_equal "http", env['rack.url_scheme']
104
- res = @lint.call(env)
104
+ assert_kind_of Array, @lint.call(env)
105
105
  end
106
106
 
107
107
  def test_rack_lint_get
@@ -109,7 +109,7 @@ class RequestTest < Test::Unit::TestCase
109
109
  env = @request.read(client)
110
110
  assert_equal "http", env['rack.url_scheme']
111
111
  assert_equal '127.0.0.1', env['REMOTE_ADDR']
112
- res = @lint.call(env)
112
+ assert_kind_of Array, @lint.call(env)
113
113
  end
114
114
 
115
115
  def test_no_content_stringio
@@ -143,7 +143,7 @@ class RequestTest < Test::Unit::TestCase
143
143
  "abcde")
144
144
  env = @request.read(client)
145
145
  assert ! env.include?(:http_body)
146
- res = @lint.call(env)
146
+ assert_kind_of Array, @lint.call(env)
147
147
  end
148
148
 
149
149
  def test_rack_lint_big_put
@@ -177,6 +177,6 @@ class RequestTest < Test::Unit::TestCase
177
177
  }
178
178
  assert_nil env['rack.input'].read(bs)
179
179
  env['rack.input'].rewind
180
- res = @lint.call(env)
180
+ assert_kind_of Array, @lint.call(env)
181
181
  end
182
182
  end