unicorn 3.6.2 → 3.7.0

data/.document

@@ -17,3 +17,4 @@ unicorn_1
 unicorn_rails_1
 ISSUES
 Sandbox
+Links

data/Documentation/GNUmakefile

@@ -1,7 +1,7 @@
 all::
 
 PANDOC = pandoc
-PANDOC_OPTS = -f markdown --email-obfuscation=none --sanitize-html
+PANDOC_OPTS = -f markdown --email-obfuscation=none
 pandoc = $(PANDOC) $(PANDOC_OPTS)
 pandoc_html = $(pandoc) --toc -t html --no-wrap
 

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v3.6.2.GIT
+DEF_VER=v3.7.0.GIT
 
 LF='
 '

data/GNUmakefile

@@ -164,7 +164,7 @@ pkg_extra := GIT-VERSION-FILE ChangeLog LATEST NEWS \
 ChangeLog: GIT-VERSION-FILE .wrongdoc.yml
 	wrongdoc prepare
 
-.manifest: ChangeLog $(ext)/unicorn_http.c
+.manifest: ChangeLog $(ext)/unicorn_http.c man
 	(git ls-files && for i in $@ $(pkg_extra); do echo $$i; done) | \
 	  LC_ALL=C sort > $@+
 	cmp $@+ $@ || mv $@+ $@
@@ -176,7 +176,7 @@ doc: .document $(ext)/unicorn_http.c man html .wrongdoc.yml
 	$(RM) -r doc
 	wrongdoc all
 	install -m644 COPYING doc/COPYING
-	install -m644 $(shell grep '^[A-Z]' .document) doc/
+	install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
 	install -m644 $(man1_paths) doc/
 	tar cf - $$(git ls-files examples/) | (cd doc && tar xf -)
 	$(RM) $(man1_rdoc)

data/HACKING

@@ -107,6 +107,17 @@ git itself.  See the Documentation/SubmittingPatches document
 distributed with git on on patch submission guidelines to follow.  Just
 don't email the git mailing list or maintainer with Unicorn patches :)
 
+== Building a Gem
+
+In order to build the gem, you must install the following components:
+
+ * wrongdoc
+ * pandoc
+
+You can build the Unicorn gem with the following command:
+
+  gmake gem
+
 == Running Development Versions
 
 It is easy to install the contents of your git working directory:

data/Links

@@ -0,0 +1,53 @@
+= Related Projects
+
+If you're interested in \Unicorn, you may be interested in some of the projects
+listed below.  If you have any links to add/change/remove, please tell us at
+mailto:mongrel-unicorn@rubyforge.org!
+
+== Disclaimer
+
+The \Unicorn project is not responsible for the content in these links.
+Furthermore, the \Unicorn project has never, does not and will never endorse:
+
+* any for-profit entities or services
+* any non-{Free Software}[http://www.gnu.org/philosophy/free-sw.html]
+
+The existence of these links does not imply endorsement of any entities
+or services behind them.
+
+=== For use with \Unicorn
+
+* {Bluepill}[https://github.com/arya/bluepill] -
+  a simple process monitoring tool written in Ruby
+
+* {golden_brindle}[https://github.com/simonoff/golden_brindle] - tool to
+  manage multiple \Unicorn instances/applications on a single server
+
+* {raindrops}[http://raindrops.bogomips.org/] - real-time stats for
+  preforking Rack servers
+
+=== \Unicorn is written to work with
+
+* {Rack}[http://rack.rubyforge.org/] - a minimal interface between webservers
+  supporting Ruby and Ruby frameworks
+
+* {Ruby}[http://ruby-lang.org/] - the programming language of Rack and \Unicorn
+
+* {nginx}[http://nginx.org/] - the reverse proxy for use with \Unicorn
+
+* {kgio}[http://bogomips.org/kgio/] - the I/O library written for \Unicorn
+
+=== Derivatives
+
+* {Green Unicorn}[http://gunicorn.org/] - a Python version of \Unicorn
+
+* {Rainbows!}[http://rainbows.rubyforge.org/] - \Unicorn for sleepy
+  apps and slow clients.
+
+=== Prior Work
+
+* {Mongrel}[http://mongrel.rubyforge.org/] - the awesome webserver \Unicorn is
+  based on
+
+* {david}[http://bogomips.org/david.git] - a tool to explain why you need
+  nginx in front of \Unicorn

data/PHILOSOPHY

@@ -1,17 +1,17 @@
-= The Philosophy Behind Unicorn
+= The Philosophy Behind unicorn
 
-Being a server that only runs on Unix-like platforms, Unicorn is
+Being a server that only runs on Unix-like platforms, unicorn is
 strongly tied to the Unix philosophy of doing one thing and (hopefully)
-doing it well.  Despite using HTTP, Unicorn is strictly a _backend_
+doing it well.  Despite using HTTP, unicorn is strictly a _backend_
 application server for running Rack-based Ruby applications.
 
 == Avoid Complexity
 
-Instead of attempting to be efficient at serving slow clients, Unicorn
+Instead of attempting to be efficient at serving slow clients, unicorn
 relies on a buffering reverse proxy to efficiently deal with slow
 clients.
 
-Unicorn uses an old-fashioned preforking worker model with blocking I/O.
+unicorn uses an old-fashioned preforking worker model with blocking I/O.
 Our processing model is the antithesis of more modern (and theoretically
 more efficient) server processing models using threads or non-blocking
 I/O with events.
@@ -19,14 +19,14 @@ I/O with events.
 === Threads and Events Are Hard
 
 ...to many developers.  Reasons for this is beyond the scope of this
-document.  Unicorn avoids concurrency within each worker process so you
+document.  unicorn avoids concurrency within each worker process so you
 have fewer things to worry about when developing your application.  Of
-course Unicorn can use multiple worker processes to utilize multiple
+course unicorn can use multiple worker processes to utilize multiple
 CPUs or spindles.  Applications can still use threads internally, however.
 
 == Slow Clients Are Problematic
 
-Most benchmarks we've seen don't tell you this, and Unicorn doesn't
+Most benchmarks we've seen don't tell you this, and unicorn doesn't
 care about slow clients... but <i>you</i> should.
 
 A "slow client" can be any client outside of your datacenter.  Network
@@ -37,71 +37,71 @@ Persistent connections were introduced in HTTP/1.1 reduce latency from
 connection establishment and TCP slow start.  They also waste server
 resources when clients are idle.
 
-Persistent connections mean one of the Unicorn worker processes
+Persistent connections mean one of the unicorn worker processes
 (depending on your application, it can be very memory hungry) would
 spend a significant amount of its time idle keeping the connection alive
 <i>and not doing anything else</i>.  Being single-threaded and using
 blocking I/O, a worker cannot serve other clients while keeping a
-connection alive.  Thus Unicorn does not implement persistent
+connection alive.  Thus unicorn does not implement persistent
 connections.
 
 If your application responses are larger than the socket buffer or if
 you're handling large requests (uploads), worker processes will also be
 bottlenecked by the speed of the *client* connection.  You should
-not allow Unicorn to serve clients outside of your local network.
+not allow unicorn to serve clients outside of your local network.
 
 == Application Concurrency != Network Concurrency
 
 Performance is asymmetric across the different subsystems of the machine
 and parts of the network.  CPUs and main memory can process gigabytes of
 data in a second; clients on the Internet are usually only capable of a
-tiny fraction of that.  Unicorn deployments should avoid dealing with
+tiny fraction of that.  unicorn deployments should avoid dealing with
 slow clients directly and instead rely on a reverse proxy to shield it
 from the effects of slow I/O.
 
 == Improved Performance Through Reverse Proxying
 
-By acting as a buffer to shield Unicorn from slow I/O, a reverse proxy
+By acting as a buffer to shield unicorn from slow I/O, a reverse proxy
 will inevitably incur overhead in the form of extra data copies.
 However, as I/O within a local network is fast (and faster still
 with local sockets), this overhead is neglible for the vast majority
 of HTTP requests and responses.
 
-The ideal reverse proxy complements the weaknesses of Unicorn.
-A reverse proxy for Unicorn should meet the following requirements:
-
-  1. It should fully buffer all HTTP requests (and large responses).
-     Each request should be "corked" in the reverse proxy and sent
-     as fast as possible to the backend Unicorn processes.  This is
-     the most important feature to look for when choosing a
-     reverse proxy for Unicorn.
-
-  2. It should spend minimal time in userspace.  Network (and disk) I/O
-     are system-level tasks and usually managed by the kernel.
-     This may change if userspace TCP stacks become more popular in the
-     future; but the reverse proxy should not waste time with
-     application-level logic.  These concerns should be separated
-
-  3. It should avoid context switches and CPU scheduling overhead.
-     In many (most?) cases, network devices and their interrupts are
-     only be handled by one CPU at a time.  It should avoid contention
-     within the system by serializing all network I/O into one (or few)
-     userspace procceses.  Network I/O is not a CPU-intensive task and
-     it is not helpful to use multiple CPU cores (at least not for GigE).
-
-  4. It should efficiently manage persistent connections (and
-     pipelining) to slow clients.  If you care to serve slow clients
-     outside your network, then these features of HTTP/1.1 will help.
-
-  5. It should (optionally) serve static files.  If you have static
-     files on your site (especially large ones), they are far more
-     efficiently served with as few data copies as possible (e.g. with
-     sendfile() to completely avoid copying the data to userspace).
+The ideal reverse proxy complements the weaknesses of unicorn.
+A reverse proxy for unicorn should meet the following requirements:
+
+1. It should fully buffer all HTTP requests (and large responses).
+   Each request should be "corked" in the reverse proxy and sent
+   as fast as possible to the backend unicorn processes.  This is
+   the most important feature to look for when choosing a
+   reverse proxy for unicorn.
+
+2. It should spend minimal time in userspace.  Network (and disk) I/O
+   are system-level tasks and usually managed by the kernel.
+   This may change if userspace TCP stacks become more popular in the
+   future; but the reverse proxy should not waste time with
+   application-level logic.  These concerns should be separated
+
+3. It should avoid context switches and CPU scheduling overhead.
+   In many (most?) cases, network devices and their interrupts are
+   only be handled by one CPU at a time.  It should avoid contention
+   within the system by serializing all network I/O into one (or few)
+   userspace procceses.  Network I/O is not a CPU-intensive task and
+   it is not helpful to use multiple CPU cores (at least not for GigE).
+
+4. It should efficiently manage persistent connections (and
+   pipelining) to slow clients.  If you care to serve slow clients
+   outside your network, then these features of HTTP/1.1 will help.
+
+5. It should (optionally) serve static files.  If you have static
+   files on your site (especially large ones), they are far more
+   efficiently served with as few data copies as possible (e.g. with
+   sendfile() to completely avoid copying the data to userspace).
 
 nginx is the only (Free) solution we know of that meets the above
 requirements.
 
-Indeed, the folks behind Unicorn have deployed nginx as a reverse-proxy not
+Indeed, the folks behind unicorn have deployed nginx as a reverse-proxy not
 only for Ruby applications, but also for production applications running
 Apache/mod_perl, Apache/mod_php and Apache Tomcat.  In every single
 case, performance improved because application servers were able to use
@@ -129,17 +129,17 @@ that is not the Unix way.
 
 == Just Worse in Some Cases
 
-Unicorn is not suited for all applications.  Unicorn is optimized for
+unicorn is not suited for all applications.  unicorn is optimized for
 applications that are CPU/memory/disk intensive and spend little time
 waiting on external resources (e.g. a database server or external API).
 
-Unicorn is highly inefficient for Comet/reverse-HTTP/push applications
+unicorn is highly inefficient for Comet/reverse-HTTP/push applications
 where the HTTP connection spends a large amount of time idle.
 Nevertheless, the ease of troubleshooting, debugging, and management of
-Unicorn may still outweigh the drawbacks for these applications.
+unicorn may still outweigh the drawbacks for these applications.
 
 The {Rainbows!}[http://rainbows.rubyforge.org/] aims to fill the gap for
-odd corner cases where the nginx + Unicorn combination is not enough.
+odd corner cases where the nginx + unicorn combination is not enough.
 While Rainbows! management/administration is largely identical to
-Unicorn, Rainbows! is far more ambitious and has seen little real-world
+unicorn, Rainbows! is far more ambitious and has seen little real-world
 usage.

data/Sandbox

@@ -3,7 +3,7 @@
 Since unicorn includes executables and is usually used to start a Ruby
 process, there are certain caveats to using it with tools that sandbox
 RubyGems installations such as
-{Bundler}[http://github.com/carlhuda/bundler] or
+{Bundler}[http://gembundler.com/] or
 {Isolate}[http://github.com/jbarnette/isolate].
 
 == General deployment
@@ -45,11 +45,20 @@ This is no longer be an issue as of bundler 0.9.17
 
 ref: http://mid.gmane.org/8FC34B23-5994-41CC-B5AF-7198EF06909E@tramchase.com
 
+=== BUNDLE_GEMFILE for Capistrano users
+
+You may need to set or reset the BUNDLE_GEMFILE environment variable in
+the before_exec hook:
+
+        before_exec do |server|
+          ENV["BUNDLE_GEMFILE"] = "/path/to/app/current/Gemfile"
+        end
+
 === Other ENV pollution issues
 
-You may need to set or reset BUNDLE_GEMFILE, GEM_HOME, GEM_PATH and PATH
-environment variables in the before_exec hook as illustrated by
-http://gist.github.com/534668
+If you're using an older Bundler version (0.9.x), you may need to set or
+reset GEM_HOME, GEM_PATH and PATH environment variables in the
+before_exec hook as illustrated by http://gist.github.com/534668
 
 == Isolate
 

data/examples/nginx.conf

@@ -87,6 +87,14 @@ http {
     # listen 80 default deferred; # for Linux
     # listen 80 default accept_filter=httpready; # for FreeBSD
 
+    # If you have IPv6, you'll likely want to have two separate listeners.
+    # One on IPv4 only (the default), and another on IPv6 only instead
+    # of a single dual-stack listener.  A dual-stack listener will make
+    # for ugly IPv4 addresses in $remote_addr (e.g ":ffff:10.0.0.1"
+    # instead of just "10.0.0.1") and potentially trigger bugs in
+    # some software.
+    # listen [::]:80 ipv6only=on; # deferred or accept_filter recommended
+
     client_max_body_size 4G;
     server_name _;
 

data/ext/unicorn_http/ext_help.h

@@ -36,6 +36,22 @@ static void rb_18_str_set_len(VALUE str, long len)
 #  endif
 #endif /* ! defined(OFFT2NUM) */
 
+#if !defined(SIZET2NUM)
+#  if SIZEOF_SIZE_T == SIZEOF_LONG
+#    define SIZET2NUM(n) ULONG2NUM(n)
+#  else
+#    define SIZET2NUM(n) ULL2NUM(n)
+#  endif
+#endif /* ! defined(SIZET2NUM) */
+
+#if !defined(NUM2SIZET)
+#  if SIZEOF_SIZE_T == SIZEOF_LONG
+#    define NUM2SIZET(n) ((size_t)NUM2ULONG(n))
+#  else
+#    define NUM2SIZET(n) ((size_t)NUM2ULL(n))
+#  endif
+#endif /* ! defined(NUM2SIZET) */
+
 #ifndef HAVE_RB_STR_MODIFY
 #  define rb_str_modify(x) do {} while (0)
 #endif /* ! defined(HAVE_RB_STR_MODIFY) */

data/ext/unicorn_http/extconf.rb

@@ -2,6 +2,7 @@
 require 'mkmf'
 
 have_macro("SIZEOF_OFF_T", "ruby.h") or check_sizeof("off_t", "sys/types.h")
+have_macro("SIZEOF_SIZE_T", "ruby.h") or check_sizeof("size_t", "sys/types.h")
 have_macro("SIZEOF_LONG", "ruby.h") or check_sizeof("long", "sys/types.h")
 have_func("rb_str_set_len", "ruby.h")
 have_func("gmtime_r", "time.h")

data/ext/unicorn_http/global_variables.h

@@ -1,6 +1,8 @@
 #ifndef global_variables_h
 #define global_variables_h
 static VALUE eHttpParserError;
+static VALUE e413;
+static VALUE e414;
 
 static VALUE g_rack_url_scheme;
 static VALUE g_request_method;
@@ -35,7 +37,7 @@ static VALUE g_http_11;
   static const char * const MAX_##N##_LENGTH_ERR = \
     "HTTP element " # N  " is longer than the " # length " allowed length."
 
-NORETURN(static void parser_error(const char *));
+NORETURN(static void parser_raise(VALUE klass, const char *));
 
 /**
  * Validates the max length of given input and throws an HttpParserError
@@ -43,7 +45,12 @@ NORETURN(static void parser_error(const char *));
  */
 #define VALIDATE_MAX_LENGTH(len, N) do { \
   if (len > MAX_##N##_LENGTH) \
-    parser_error(MAX_##N##_LENGTH_ERR); \
+    parser_raise(eHttpParserError, MAX_##N##_LENGTH_ERR); \
+} while (0)
+
+#define VALIDATE_MAX_URI_LENGTH(len, N) do { \
+  if (len > MAX_##N##_LENGTH) \
+    parser_raise(e414, MAX_##N##_LENGTH_ERR); \
 } while (0)
 
 /** Defines global strings in the init method. */
@@ -59,7 +66,6 @@ DEF_MAX_LENGTH(REQUEST_URI, 1024 * 12);
 DEF_MAX_LENGTH(FRAGMENT, 1024); /* Don't know if this length is specified somewhere or not */
 DEF_MAX_LENGTH(REQUEST_PATH, 1024);
 DEF_MAX_LENGTH(QUERY_STRING, (1024 * 10));
-DEF_MAX_LENGTH(HEADER, (1024 * (80 + 32)));
 
 static void init_globals(void)
 {

data/ext/unicorn_http/unicorn_http.rl

@@ -82,6 +82,14 @@ static VALUE xftrust(VALUE self)
   return trust_x_forward;
 }
 
+static size_t MAX_HEADER_LEN = 1024 * (80 + 32); /* same as Mongrel */
+
+/* this is only intended for use with Rainbows! */
+static VALUE set_maxhdrlen(VALUE self, VALUE len)
+{
+  return SIZET2NUM(MAX_HEADER_LEN = NUM2SIZET(len));
+}
+
 /* keep this small for Rainbows! since every client has one */
 struct http_parser {
   int cs; /* Ragel internal state */
@@ -106,17 +114,17 @@ struct http_parser {
   } len;
 };
 
-static ID id_clear;
+static ID id_clear, id_set_backtrace;
 
 static void finalize_header(struct http_parser *hp);
 
-static void parser_error(const char *msg)
+static void parser_raise(VALUE klass, const char *msg)
 {
-  VALUE exc = rb_exc_new2(eHttpParserError, msg);
+  VALUE exc = rb_exc_new2(klass, msg);
   VALUE bt = rb_ary_new();
 
-  rb_funcall(exc, rb_intern("set_backtrace"), 1, bt);
-  rb_exc_raise(exc);
+	rb_funcall(exc, id_set_backtrace, 1, bt);
+	rb_exc_raise(exc);
 }
 
 #define REMAINING (unsigned long)(pe - p)
@@ -124,12 +132,27 @@ static void parser_error(const char *msg)
 #define MARK(M,FPC) (hp->M = (FPC) - buffer)
 #define PTR_TO(F) (buffer + hp->F)
 #define STR_NEW(M,FPC) rb_str_new(PTR_TO(M), LEN(M, FPC))
+#define STRIPPED_STR_NEW(M,FPC) stripped_str_new(PTR_TO(M), LEN(M, FPC))
 
 #define HP_FL_TEST(hp,fl) ((hp)->flags & (UH_FL_##fl))
 #define HP_FL_SET(hp,fl) ((hp)->flags |= (UH_FL_##fl))
 #define HP_FL_UNSET(hp,fl) ((hp)->flags &= ~(UH_FL_##fl))
 #define HP_FL_ALL(hp,fl) (HP_FL_TEST(hp, fl) == (UH_FL_##fl))
 
+static int is_lws(char c)
+{
+  return (c == ' ' || c == '\t');
+}
+
+static VALUE stripped_str_new(const char *str, long len)
+{
+  long end;
+
+  for (end = len - 1; end >= 0 && is_lws(str[end]); end--);
+
+  return rb_str_new(str, end + 1);
+}
+
 /*
  * handles values of the "Connection:" header, keepalive is implied
  * for HTTP/1.1 but needs to be explicitly enabled with HTTP/1.0
@@ -186,35 +209,43 @@ http_version(struct http_parser *hp, const char *ptr, size_t len)
 static inline void hp_invalid_if_trailer(struct http_parser *hp)
 {
   if (HP_FL_TEST(hp, INTRAILER))
-    parser_error("invalid Trailer");
+    parser_raise(eHttpParserError, "invalid Trailer");
 }
 
 static void write_cont_value(struct http_parser *hp,
                              char *buffer, const char *p)
 {
   char *vptr;
+  long end;
+  long len = LEN(mark, p);
+  long cont_len;
 
   if (hp->cont == Qfalse)
-     parser_error("invalid continuation line");
+     parser_raise(eHttpParserError, "invalid continuation line");
   if (NIL_P(hp->cont))
      return; /* we're ignoring this header (probably Host:) */
 
   assert(TYPE(hp->cont) == T_STRING && "continuation line is not a string");
   assert(hp->mark > 0 && "impossible continuation line offset");
 
-  if (LEN(mark, p) == 0)
+  if (len == 0)
     return;
 
-  if (RSTRING_LEN(hp->cont) > 0)
+  cont_len = RSTRING_LEN(hp->cont);
+  if (cont_len > 0) {
     --hp->mark;
-
+    len = LEN(mark, p);
+  }
   vptr = PTR_TO(mark);
 
-  if (RSTRING_LEN(hp->cont) > 0) {
+  /* normalize tab to space */
+  if (cont_len > 0) {
     assert((' ' == *vptr || '\t' == *vptr) && "invalid leading white space");
     *vptr = ' ';
   }
-  rb_str_buf_cat(hp->cont, vptr, LEN(mark, p));
+
+  for (end = len - 1; end >= 0 && is_lws(vptr[end]); end--);
+  rb_str_buf_cat(hp->cont, vptr, end + 1);
 }
 
 static void write_value(struct http_parser *hp,
@@ -225,7 +256,7 @@ static void write_value(struct http_parser *hp,
   VALUE e;
 
   VALIDATE_MAX_LENGTH(LEN(mark, p), FIELD_VALUE);
-  v = LEN(mark, p) == 0 ? rb_str_buf_new(128) : STR_NEW(mark, p);
+  v = LEN(mark, p) == 0 ? rb_str_buf_new(128) : STRIPPED_STR_NEW(mark, p);
   if (NIL_P(f)) {
     const char *field = PTR_TO(start.field);
     size_t flen = hp->s.field_len;
@@ -246,7 +277,7 @@ static void write_value(struct http_parser *hp,
   } else if (f == g_content_length) {
     hp->len.content = parse_length(RSTRING_PTR(v), RSTRING_LEN(v));
     if (hp->len.content < 0)
-      parser_error("invalid Content-Length");
+      parser_raise(eHttpParserError, "invalid Content-Length");
     if (hp->len.content != 0)
       HP_FL_SET(hp, HASBODY);
     hp_invalid_if_trailer(hp);
@@ -301,7 +332,7 @@ static void write_value(struct http_parser *hp,
   action request_uri {
     VALUE str;
 
-    VALIDATE_MAX_LENGTH(LEN(mark, fpc), REQUEST_URI);
+    VALIDATE_MAX_URI_LENGTH(LEN(mark, fpc), REQUEST_URI);
     str = rb_hash_aset(hp->env, g_request_uri, STR_NEW(mark, fpc));
     /*
      * "OPTIONS * HTTP/1.1\r\n" is a valid request, but we can't have '*'
@@ -314,19 +345,19 @@ static void write_value(struct http_parser *hp,
     }
   }
   action fragment {
-    VALIDATE_MAX_LENGTH(LEN(mark, fpc), FRAGMENT);
+    VALIDATE_MAX_URI_LENGTH(LEN(mark, fpc), FRAGMENT);
     rb_hash_aset(hp->env, g_fragment, STR_NEW(mark, fpc));
   }
   action start_query {MARK(start.query, fpc); }
   action query_string {
-    VALIDATE_MAX_LENGTH(LEN(start.query, fpc), QUERY_STRING);
+    VALIDATE_MAX_URI_LENGTH(LEN(start.query, fpc), QUERY_STRING);
     rb_hash_aset(hp->env, g_query_string, STR_NEW(start.query, fpc));
   }
   action http_version { http_version(hp, PTR_TO(mark), LEN(mark, fpc)); }
   action request_path {
     VALUE val;
 
-    VALIDATE_MAX_LENGTH(LEN(mark, fpc), REQUEST_PATH);
+    VALIDATE_MAX_URI_LENGTH(LEN(mark, fpc), REQUEST_PATH);
     val = rb_hash_aset(hp->env, g_request_path, STR_NEW(mark, fpc));
 
     /* rack says PATH_INFO must start with "/" or be empty */
@@ -336,7 +367,7 @@ static void write_value(struct http_parser *hp,
   action add_to_chunk_size {
     hp->len.chunk = step_incr(hp->len.chunk, fc, 16);
     if (hp->len.chunk < 0)
-      parser_error("invalid chunk size");
+      parser_raise(eHttpParserError, "invalid chunk size");
   }
   action header_done {
     finalize_header(hp);
@@ -677,7 +708,8 @@ static VALUE HttpParser_parse(VALUE self)
   }
 
   http_parser_execute(hp, RSTRING_PTR(data), RSTRING_LEN(data));
-  VALIDATE_MAX_LENGTH(hp->offset, HEADER);
+  if (hp->offset > MAX_HEADER_LEN)
+    parser_raise(e413, "HTTP header is too large");
 
   if (hp->cs == http_parser_first_final ||
       hp->cs == http_parser_en_ChunkedBody) {
@@ -690,11 +722,32 @@ static VALUE HttpParser_parse(VALUE self)
   }
 
   if (hp->cs == http_parser_error)
-    parser_error("Invalid HTTP format, parsing fails.");
+    parser_raise(eHttpParserError, "Invalid HTTP format, parsing fails.");
 
   return Qnil;
 }
 
+/**
+ * Document-method: parse
+ * call-seq:
+ *    parser.add_parse(buffer) => env or nil
+ *
+ * adds the contents of +buffer+ to the internal buffer and attempts to
+ * continue parsing.  Returns the +env+ Hash on success or nil if more
+ * data is needed.
+ *
+ * Raises HttpParserError if there are parsing errors.
+ */
+static VALUE HttpParser_add_parse(VALUE self, VALUE buffer)
+{
+  struct http_parser *hp = data_get(self);
+
+  Check_Type(buffer, T_STRING);
+  rb_str_buf_append(hp->buf, buffer);
+
+  return HttpParser_parse(self);
+}
+
 /**
  * Document-method: trailers
  * call-seq:
@@ -825,6 +878,7 @@ static VALUE HttpParser_filter_body(VALUE self, VALUE buf, VALUE data)
   dlen = RSTRING_LEN(data);
 
   StringValue(buf);
+  rb_str_modify(buf);
   rb_str_resize(buf, dlen); /* we can never copy more than dlen bytes */
   OBJ_TAINT(buf); /* keep weirdo $SAFE users happy */
 
@@ -835,7 +889,7 @@ static VALUE HttpParser_filter_body(VALUE self, VALUE buf, VALUE data)
       hp->buf = data;
       http_parser_execute(hp, dptr, dlen);
       if (hp->cs == http_parser_error)
-        parser_error("Invalid HTTP format, parsing fails.");
+        parser_raise(eHttpParserError, "Invalid HTTP format, parsing fails.");
 
       assert(hp->s.dest_offset <= hp->offset &&
              "destination buffer overflow");
@@ -883,6 +937,10 @@ void Init_unicorn_http(void)
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
   eHttpParserError =
          rb_define_class_under(mUnicorn, "HttpParserError", rb_eIOError);
+  e413 = rb_define_class_under(mUnicorn, "RequestEntityTooLargeError",
+                               eHttpParserError);
+  e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
+                               eHttpParserError);
 
   init_globals();
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
@@ -890,6 +948,7 @@ void Init_unicorn_http(void)
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
   rb_define_method(cHttpParser, "reset", HttpParser_reset, 0);
   rb_define_method(cHttpParser, "parse", HttpParser_parse, 0);
+  rb_define_method(cHttpParser, "add_parse", HttpParser_add_parse, 1);
   rb_define_method(cHttpParser, "headers", HttpParser_headers, 2);
   rb_define_method(cHttpParser, "trailers", HttpParser_headers, 2);
   rb_define_method(cHttpParser, "filter_body", HttpParser_filter_body, 2);
@@ -924,6 +983,7 @@ void Init_unicorn_http(void)
   rb_define_singleton_method(cHttpParser, "keepalive_requests=", set_ka_req, 1);
   rb_define_singleton_method(cHttpParser, "trust_x_forwarded=", set_xftrust, 1);
   rb_define_singleton_method(cHttpParser, "trust_x_forwarded?", xftrust, 0);
+  rb_define_singleton_method(cHttpParser, "max_header_len=", set_maxhdrlen, 1);
 
   init_common_fields();
   SET_GLOBAL(g_http_host, "HOST");
@@ -932,6 +992,7 @@ void Init_unicorn_http(void)
   SET_GLOBAL(g_content_length, "CONTENT_LENGTH");
   SET_GLOBAL(g_http_connection, "CONNECTION");
   id_clear = rb_intern("clear");
+  id_set_backtrace = rb_intern("set_backtrace");
   init_unicorn_httpdate();
 }
 #undef SET_GLOBAL

data/lib/unicorn/configurator.rb

@@ -281,6 +281,22 @@ class Unicorn::Configurator
   #
   #   Default: +true+ in \Unicorn 3.4+, +false+ in Rainbows!
   #
+  # [:ipv6only => true or false]
+  #
+  #   This option makes IPv6-capable TCP listeners IPv6-only and unable
+  #   to receive IPv4 queries on dual-stack systems.  A separate IPv4-only
+  #   listener is required if this is true.
+  #
+  #   This option is only available for Ruby 1.9.2 and later.
+  #
+  #   Enabling this option for the IPv6-only listener and having a
+  #   separate IPv4 listener is recommended if you wish to support IPv6
+  #   on the same TCP port.  Otherwise, the value of \env[\"REMOTE_ADDR\"]
+  #   will appear as an ugly IPv4-mapped-IPv6 address for IPv4 clients
+  #   (e.g ":ffff:10.0.0.1" instead of just "10.0.0.1").
+  #
+  #   Default: Operating-system dependent
+  #
   # [:tries => Integer]
   #
   #   Times to retry binding a socket if it is already in use
@@ -358,7 +374,7 @@ class Unicorn::Configurator
         Integer === value or
           raise ArgumentError, "not an integer: #{key}=#{value.inspect}"
       end
-      [ :tcp_nodelay, :tcp_nopush ].each do |key|
+      [ :tcp_nodelay, :tcp_nopush, :ipv6only ].each do |key|
         (value = options[key]).nil? and next
         TrueClass === value || FalseClass === value or
           raise ArgumentError, "not boolean: #{key}=#{value.inspect}"

data/lib/unicorn/const.rb

@@ -8,8 +8,8 @@
 # improve things much compared to constants.
 module Unicorn::Const
 
-  # The current version of Unicorn, currently 3.6.2
-  UNICORN_VERSION = "3.6.2"
+  # The current version of Unicorn, currently 3.7.0
+  UNICORN_VERSION = "3.7.0"
 
   # default TCP listen host address (0.0.0.0, all interfaces)
   DEFAULT_HOST = "0.0.0.0"
@@ -25,12 +25,14 @@ module Unicorn::Const
 
   # Maximum request body size before it is moved out of memory and into a
   # temporary file for reading (112 kilobytes).  This is the default
-  # value of of client_body_buffer_size.
+  # value of client_body_buffer_size.
   MAX_BODY = 1024 * 112
 
   # :stopdoc:
   # common errors we'll send back
   ERROR_400_RESPONSE = "HTTP/1.1 400 Bad Request\r\n\r\n"
+  ERROR_414_RESPONSE = "HTTP/1.1 414 Request-URI Too Long\r\n\r\n"
+  ERROR_413_RESPONSE = "HTTP/1.1 413 Request Entity Too Large\r\n\r\n"
   ERROR_500_RESPONSE = "HTTP/1.1 500 Internal Server Error\r\n\r\n"
   EXPECT_100_RESPONSE = "HTTP/1.1 100 Continue\r\n\r\n"
 

data/lib/unicorn/http_request.rb

@@ -68,9 +68,7 @@ class Unicorn::HttpParser
     if parse.nil?
       # Parser is not done, queue up more data to read and continue parsing
       # an Exception thrown from the parser will throw us out of the loop
-      begin
-        buf << socket.kgio_read!(16384)
-      end while parse.nil?
+      false until add_parse(socket.kgio_read!(16384))
     end
     e[RACK_INPUT] = 0 == content_length ?
                     NULL_IO : @@input_class.new(socket, self)

data/lib/unicorn/http_server.rb

@@ -23,9 +23,6 @@ class Unicorn::HttpServer
   # backwards compatibility with 1.x
   Worker = Unicorn::Worker
 
-  # prevents IO objects in here from being GC-ed
-  IO_PURGATORY = []
-
   # all bound listener sockets
   LISTENERS = []
 
@@ -527,6 +524,10 @@ class Unicorn::HttpServer
     msg = case e
     when EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::EINVAL,Errno::EBADF
       Unicorn::Const::ERROR_500_RESPONSE
+    when Unicorn::RequestURITooLongError
+      Unicorn::Const::ERROR_414_RESPONSE
+    when Unicorn::RequestEntityTooLargeError
+      Unicorn::Const::ERROR_413_RESPONSE
     when Unicorn::HttpParserError # try to tell the client they're bad
       Unicorn::Const::ERROR_400_RESPONSE
     else

data/lib/unicorn/socket_helper.rb

@@ -4,9 +4,12 @@ require 'socket'
 
 module Unicorn
   module SocketHelper
+    # :stopdoc:
     include Socket::Constants
 
-    # :stopdoc:
+    # prevents IO objects in here from being GC-ed
+    IO_PURGATORY = []
+
     # internal interface, only used by Rainbows!/Zbatery
     DEFAULTS = {
       # The semantics for TCP_DEFER_ACCEPT changed in Linux 2.6.32+
@@ -136,8 +139,9 @@ module Unicorn
         ensure
           File.umask(old_umask)
         end
-      elsif /\A(\d+\.\d+\.\d+\.\d+):(\d+)\z/ =~ address ||
-            /\A\[([a-fA-F0-9:]+)\]:(\d+)\z/ =~ address
+      elsif /\A\[([a-fA-F0-9:]+)\]:(\d+)\z/ =~ address
+        new_ipv6_server($1, $2.to_i, opt)
+      elsif /\A(\d+\.\d+\.\d+\.\d+):(\d+)\z/ =~ address
         Kgio::TCPServer.new($1, $2.to_i)
       else
         raise ArgumentError, "Don't know how to bind: #{address}"
@@ -146,6 +150,18 @@ module Unicorn
       sock
     end
 
+    def new_ipv6_server(addr, port, opt)
+      opt.key?(:ipv6only) or return Kgio::TCPServer.new(addr, port)
+      defined?(IPV6_V6ONLY) or
+        abort "Socket::IPV6_V6ONLY not defined, upgrade Ruby and/or your OS"
+      sock = Socket.new(AF_INET6, SOCK_STREAM, 0)
+      sock.setsockopt(IPPROTO_IPV6, IPV6_V6ONLY, opt[:ipv6only] ? 1 : 0)
+      sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+      sock.bind(Socket.pack_sockaddr_in(port, addr))
+      IO_PURGATORY << sock
+      Kgio::TCPServer.for_fd(sock.fileno)
+    end
+
     # returns rfc2732-style (e.g. "[::1]:666") addresses for IPv6
     def tcp_name(sock)
       port, addr = Socket.unpack_sockaddr_in(sock.getsockname)

data/script/isolate_for_tests

@@ -35,6 +35,7 @@ File.rename("#{dst}.#$$", dst)
   opts[:path] = "tmp/isolate/rails-#{rails_ver}"
   pid = fork do
     Isolate.now!(opts) do
+      gem 'rake', '0.8.7'
       gem 'rails', rails_ver
     end
   end

data/t/t0002-parser-error.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 . ./test-lib.sh
-t_plan 5 "parser error test"
+t_plan 11 "parser error test"
 
 t_begin "setup and startup" && {
 	unicorn_setup
@@ -24,6 +24,69 @@ t_begin "response should be a 400" && {
 	grep -F 'HTTP/1.1 400 Bad Request' $tmp
 }
 
+t_begin "send a huge Request URI (REQUEST_PATH > (12 * 1024))" && {
+	rm -f $tmp
+	cat $fifo > $tmp &
+	(
+		set -e
+		trap 'echo ok > $ok' EXIT
+		printf 'GET /'
+		for i in $(awk </dev/null 'BEGIN{for(i=0;i<1024;i++) print i}')
+		do
+			printf '0123456789ab'
+		done
+		printf ' HTTP/1.1\r\nHost: example.com\r\n\r\n'
+	) | socat - TCP:$listen > $fifo || :
+	test xok = x$(cat $ok)
+	wait
+}
+
+t_begin "response should be a 414 (REQUEST_PATH)" && {
+	grep -F 'HTTP/1.1 414 Request-URI Too Long' $tmp
+}
+
+t_begin "send a huge Request URI (QUERY_STRING > (10 * 1024))" && {
+	rm -f $tmp
+	cat $fifo > $tmp &
+	(
+		set -e
+		trap 'echo ok > $ok' EXIT
+		printf 'GET /hello-world?a'
+		for i in $(awk </dev/null 'BEGIN{for(i=0;i<1024;i++) print i}')
+		do
+			printf '0123456789'
+		done
+		printf ' HTTP/1.1\r\nHost: example.com\r\n\r\n'
+	) | socat - TCP:$listen > $fifo || :
+	test xok = x$(cat $ok)
+	wait
+}
+
+t_begin "response should be a 414 (QUERY_STRING)" && {
+	grep -F 'HTTP/1.1 414 Request-URI Too Long' $tmp
+}
+
+t_begin "send a huge Request URI (FRAGMENT > 1024)" && {
+	rm -f $tmp
+	cat $fifo > $tmp &
+	(
+		set -e
+		trap 'echo ok > $ok' EXIT
+		printf 'GET /hello-world#a'
+		for i in $(awk </dev/null 'BEGIN{for(i=0;i<64;i++) print i}')
+		do
+			printf '0123456789abcdef'
+		done
+		printf ' HTTP/1.1\r\nHost: example.com\r\n\r\n'
+	) | socat - TCP:$listen > $fifo || :
+	test xok = x$(cat $ok)
+	wait
+}
+
+t_begin "response should be a 414 (FRAGMENT)" && {
+	grep -F 'HTTP/1.1 414 Request-URI Too Long' $tmp
+}
+
 t_begin "server stderr should be clean" && check_stderr
 
 t_begin "term signal sent" && kill $unicorn_pid

data/t/t0019-max_header_len.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 5 "max_header_len setting (only intended for Rainbows!)"
+
+t_begin "setup and start" && {
+	unicorn_setup
+	req='GET / HTTP/1.0\r\n\r\n'
+	len=$(printf "$req" | wc -c)
+	echo Unicorn::HttpParser.max_header_len = $len >> $unicorn_config
+	unicorn -D -c $unicorn_config env.ru
+	unicorn_wait_start
+}
+
+t_begin "minimal request succeeds" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf "$req"
+		wait
+		echo ok > $ok
+	) | socat - TCP:$listen > $fifo
+	test xok = x$(cat $ok)
+
+	fgrep "HTTP/1.1 200 OK" $tmp
+}
+
+t_begin "big request fails" && {
+	rm -f $tmp
+	(
+		cat $fifo > $tmp &
+		printf 'GET /xxxxxx HTTP/1.0\r\n\r\n'
+		wait
+		echo ok > $ok
+	) | socat - TCP:$listen > $fifo
+	test xok = x$(cat $ok)
+	fgrep "HTTP/1.1 413" $tmp
+}
+
+dbgcat tmp
+
+t_begin "killing succeeds" && {
+	kill $unicorn_pid
+}
+
+t_begin "check stderr" && {
+	check_stderr
+}
+
+t_done

data/test/unit/test_http_parser.rb

@@ -258,6 +258,20 @@ class HttpParserTest < Test::Unit::TestCase
     assert_equal 'hi y x ASDF', req['HTTP_X_ASDF']
   end
 
+  def test_continuation_eats_trailing_spaces
+    parser = HttpParser.new
+    header = "GET / HTTP/1.1\r\n" \
+             "X-ASDF:      \r\n" \
+             "\t\r\n" \
+             "  b  \r\n" \
+             "  ASDF\r\n\r\n"
+    parser.buf << header
+    req = parser.env
+    assert_equal req, parser.parse
+    assert_equal '', parser.buf
+    assert_equal 'b ASDF', req['HTTP_X_ASDF']
+  end
+
   def test_continuation_with_absolute_uri_and_ignored_host_header
     parser = HttpParser.new
     header = "GET http://example.com/ HTTP/1.1\r\n" \
@@ -764,6 +778,48 @@ class HttpParserTest < Test::Unit::TestCase
 
   end
 
+  def test_leading_tab
+    parser = HttpParser.new
+    get = "GET / HTTP/1.1\r\nHost:\texample.com\r\n\r\n"
+    assert parser.add_parse(get)
+    assert_equal 'example.com', parser.env['HTTP_HOST']
+  end
+
+  def test_trailing_whitespace
+    parser = HttpParser.new
+    get = "GET / HTTP/1.1\r\nHost: example.com \r\n\r\n"
+    assert parser.add_parse(get)
+    assert_equal 'example.com', parser.env['HTTP_HOST']
+  end
+
+  def test_trailing_tab
+    parser = HttpParser.new
+    get = "GET / HTTP/1.1\r\nHost: example.com\t\r\n\r\n"
+    assert parser.add_parse(get)
+    assert_equal 'example.com', parser.env['HTTP_HOST']
+  end
+
+  def test_trailing_multiple_linear_whitespace
+    parser = HttpParser.new
+    get = "GET / HTTP/1.1\r\nHost: example.com\t \t \t\r\n\r\n"
+    assert parser.add_parse(get)
+    assert_equal 'example.com', parser.env['HTTP_HOST']
+  end
+
+  def test_embedded_linear_whitespace_ok
+    parser = HttpParser.new
+    get = "GET / HTTP/1.1\r\nX-Space: hello\t world\t \r\n\r\n"
+    assert parser.add_parse(get)
+    assert_equal "hello\t world", parser.env["HTTP_X_SPACE"]
+  end
+
+  def test_empty_header
+    parser = HttpParser.new
+    get = "GET / HTTP/1.1\r\nHost:  \r\n\r\n"
+    assert parser.add_parse(get)
+    assert_equal '', parser.env['HTTP_HOST']
+  end
+
   # so we don't  care about the portability of this test
   # if it doesn't leak on Linux, it won't leak anywhere else
   # unless your C compiler or platform is otherwise broken

data/test/unit/test_socket_helper.rb

@@ -12,6 +12,7 @@ class TestSocketHelper < Test::Unit::TestCase
     @log_tmp = Tempfile.new 'logger'
     @logger = Logger.new(@log_tmp.path)
     @test_addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
+    @test6_addr = ENV['UNICORN_TEST6_ADDR'] || '::1'
     GC.disable
   end
 
@@ -177,4 +178,11 @@ class TestSocketHelper < Test::Unit::TestCase
     assert cur > 1
   end if defined?(TCP_DEFER_ACCEPT)
 
+  def test_ipv6only
+    port = unused_port "#@test6_addr"
+    sock = bind_listen "[#@test6_addr]:#{port}", :ipv6only => true
+    cur = sock.getsockopt(:IPPROTO_IPV6, :IPV6_V6ONLY).unpack('i')[0]
+    assert_equal 1, cur
+    rescue Errno::EAFNOSUPPORT
+  end if RUBY_VERSION >= "1.9.2"
 end

data/unicorn.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |s|
   s.add_dependency(%q<rack>)
   s.add_dependency(%q<kgio>, '~> 2.3')
 
-  s.add_development_dependency('isolate', '~> 3.0.0')
+  s.add_development_dependency('isolate', '~> 3.1')
   s.add_development_dependency('wrongdoc', '~> 1.5')
 
   # s.licenses = %w(GPLv2 Ruby) # licenses= method is not in older RubyGems

metadata

@@ -5,9 +5,9 @@ version: !ruby/object:Gem::Version
   prerelease: 
   segments: 
   - 3
-  - 6
-  - 2
-  version: 3.6.2
+  - 7
+  - 0
+  version: 3.7.0
 platform: ruby
 authors: 
 - Unicorn hackers
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-30 00:00:00 Z
+date: 2011-06-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rack
@@ -54,12 +54,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
+        hash: 5
         segments: 
         - 3
-        - 0
-        - 0
-        version: 3.0.0
+        - 1
+        version: "3.1"
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -126,6 +125,7 @@ extra_rdoc_files:
 - lib/unicorn/worker.rb
 - ISSUES
 - Sandbox
+- Links
 files: 
 - .CHANGELOG.old
 - .document
@@ -150,6 +150,7 @@ files:
 - KNOWN_ISSUES
 - LATEST
 - LICENSE
+- Links
 - NEWS
 - PHILOSOPHY
 - README
@@ -272,6 +273,7 @@ files:
 - t/t0016-trust-x-forwarded-false.sh
 - t/t0017-trust-x-forwarded-true.sh
 - t/t0018-write-on-close.sh
+- t/t0019-max_header_len.sh
 - t/t0100-rack-input-tests.sh
 - t/t0116-client_body_buffer_size.sh
 - t/t0116.ru
@@ -399,7 +401,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: mongrel
-rubygems_version: 1.7.2
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: Rack HTTP server for fast clients and Unix