unicorn 3.6.0 → 4.0.0

data/ext/unicorn_http/unicorn_http.rl

@@ -82,6 +82,14 @@ static VALUE xftrust(VALUE self)
   return trust_x_forward;
 }
 
+static size_t MAX_HEADER_LEN = 1024 * (80 + 32); /* same as Mongrel */
+
+/* this is only intended for use with Rainbows! */
+static VALUE set_maxhdrlen(VALUE self, VALUE len)
+{
+  return SIZET2NUM(MAX_HEADER_LEN = NUM2SIZET(len));
+}
+
 /* keep this small for Rainbows! since every client has one */
 struct http_parser {
   int cs; /* Ragel internal state */
@@ -106,17 +114,17 @@ struct http_parser {
   } len;
 };
 
-static ID id_clear;
+static ID id_clear, id_set_backtrace;
 
 static void finalize_header(struct http_parser *hp);
 
-static void parser_error(const char *msg)
+static void parser_raise(VALUE klass, const char *msg)
 {
-  VALUE exc = rb_exc_new2(eHttpParserError, msg);
+  VALUE exc = rb_exc_new2(klass, msg);
   VALUE bt = rb_ary_new();
 
-  rb_funcall(exc, rb_intern("set_backtrace"), 1, bt);
-  rb_exc_raise(exc);
+	rb_funcall(exc, id_set_backtrace, 1, bt);
+	rb_exc_raise(exc);
 }
 
 #define REMAINING (unsigned long)(pe - p)
@@ -124,12 +132,27 @@ static void parser_error(const char *msg)
 #define MARK(M,FPC) (hp->M = (FPC) - buffer)
 #define PTR_TO(F) (buffer + hp->F)
 #define STR_NEW(M,FPC) rb_str_new(PTR_TO(M), LEN(M, FPC))
+#define STRIPPED_STR_NEW(M,FPC) stripped_str_new(PTR_TO(M), LEN(M, FPC))
 
 #define HP_FL_TEST(hp,fl) ((hp)->flags & (UH_FL_##fl))
 #define HP_FL_SET(hp,fl) ((hp)->flags |= (UH_FL_##fl))
 #define HP_FL_UNSET(hp,fl) ((hp)->flags &= ~(UH_FL_##fl))
 #define HP_FL_ALL(hp,fl) (HP_FL_TEST(hp, fl) == (UH_FL_##fl))
 
+static int is_lws(char c)
+{
+  return (c == ' ' || c == '\t');
+}
+
+static VALUE stripped_str_new(const char *str, long len)
+{
+  long end;
+
+  for (end = len - 1; end >= 0 && is_lws(str[end]); end--);
+
+  return rb_str_new(str, end + 1);
+}
+
 /*
  * handles values of the "Connection:" header, keepalive is implied
  * for HTTP/1.1 but needs to be explicitly enabled with HTTP/1.0
@@ -186,35 +209,43 @@ http_version(struct http_parser *hp, const char *ptr, size_t len)
 static inline void hp_invalid_if_trailer(struct http_parser *hp)
 {
   if (HP_FL_TEST(hp, INTRAILER))
-    parser_error("invalid Trailer");
+    parser_raise(eHttpParserError, "invalid Trailer");
 }
 
 static void write_cont_value(struct http_parser *hp,
                              char *buffer, const char *p)
 {
   char *vptr;
+  long end;
+  long len = LEN(mark, p);
+  long cont_len;
 
   if (hp->cont == Qfalse)
-     parser_error("invalid continuation line");
+     parser_raise(eHttpParserError, "invalid continuation line");
   if (NIL_P(hp->cont))
      return; /* we're ignoring this header (probably Host:) */
 
   assert(TYPE(hp->cont) == T_STRING && "continuation line is not a string");
   assert(hp->mark > 0 && "impossible continuation line offset");
 
-  if (LEN(mark, p) == 0)
+  if (len == 0)
     return;
 
-  if (RSTRING_LEN(hp->cont) > 0)
+  cont_len = RSTRING_LEN(hp->cont);
+  if (cont_len > 0) {
     --hp->mark;
-
+    len = LEN(mark, p);
+  }
   vptr = PTR_TO(mark);
 
-  if (RSTRING_LEN(hp->cont) > 0) {
+  /* normalize tab to space */
+  if (cont_len > 0) {
     assert((' ' == *vptr || '\t' == *vptr) && "invalid leading white space");
     *vptr = ' ';
   }
-  rb_str_buf_cat(hp->cont, vptr, LEN(mark, p));
+
+  for (end = len - 1; end >= 0 && is_lws(vptr[end]); end--);
+  rb_str_buf_cat(hp->cont, vptr, end + 1);
 }
 
 static void write_value(struct http_parser *hp,
@@ -225,7 +256,7 @@ static void write_value(struct http_parser *hp,
   VALUE e;
 
   VALIDATE_MAX_LENGTH(LEN(mark, p), FIELD_VALUE);
-  v = LEN(mark, p) == 0 ? rb_str_buf_new(128) : STR_NEW(mark, p);
+  v = LEN(mark, p) == 0 ? rb_str_buf_new(128) : STRIPPED_STR_NEW(mark, p);
   if (NIL_P(f)) {
     const char *field = PTR_TO(start.field);
     size_t flen = hp->s.field_len;
@@ -246,7 +277,7 @@ static void write_value(struct http_parser *hp,
   } else if (f == g_content_length) {
     hp->len.content = parse_length(RSTRING_PTR(v), RSTRING_LEN(v));
     if (hp->len.content < 0)
-      parser_error("invalid Content-Length");
+      parser_raise(eHttpParserError, "invalid Content-Length");
     if (hp->len.content != 0)
       HP_FL_SET(hp, HASBODY);
     hp_invalid_if_trailer(hp);
@@ -301,7 +332,7 @@ static void write_value(struct http_parser *hp,
   action request_uri {
     VALUE str;
 
-    VALIDATE_MAX_LENGTH(LEN(mark, fpc), REQUEST_URI);
+    VALIDATE_MAX_URI_LENGTH(LEN(mark, fpc), REQUEST_URI);
     str = rb_hash_aset(hp->env, g_request_uri, STR_NEW(mark, fpc));
     /*
      * "OPTIONS * HTTP/1.1\r\n" is a valid request, but we can't have '*'
@@ -314,19 +345,19 @@ static void write_value(struct http_parser *hp,
     }
   }
   action fragment {
-    VALIDATE_MAX_LENGTH(LEN(mark, fpc), FRAGMENT);
+    VALIDATE_MAX_URI_LENGTH(LEN(mark, fpc), FRAGMENT);
     rb_hash_aset(hp->env, g_fragment, STR_NEW(mark, fpc));
   }
   action start_query {MARK(start.query, fpc); }
   action query_string {
-    VALIDATE_MAX_LENGTH(LEN(start.query, fpc), QUERY_STRING);
+    VALIDATE_MAX_URI_LENGTH(LEN(start.query, fpc), QUERY_STRING);
     rb_hash_aset(hp->env, g_query_string, STR_NEW(start.query, fpc));
   }
   action http_version { http_version(hp, PTR_TO(mark), LEN(mark, fpc)); }
   action request_path {
     VALUE val;
 
-    VALIDATE_MAX_LENGTH(LEN(mark, fpc), REQUEST_PATH);
+    VALIDATE_MAX_URI_LENGTH(LEN(mark, fpc), REQUEST_PATH);
     val = rb_hash_aset(hp->env, g_request_path, STR_NEW(mark, fpc));
 
     /* rack says PATH_INFO must start with "/" or be empty */
@@ -336,7 +367,7 @@ static void write_value(struct http_parser *hp,
   action add_to_chunk_size {
     hp->len.chunk = step_incr(hp->len.chunk, fc, 16);
     if (hp->len.chunk < 0)
-      parser_error("invalid chunk size");
+      parser_raise(eHttpParserError, "invalid chunk size");
   }
   action header_done {
     finalize_header(hp);
@@ -598,6 +629,34 @@ static VALUE HttpParser_clear(VALUE self)
   return self;
 }
 
+/**
+ * call-seq:
+ *    parser.dechunk! => parser
+ *
+ * Resets the parser to a state suitable for dechunking response bodies
+ *
+ */
+static VALUE HttpParser_dechunk_bang(VALUE self)
+{
+  struct http_parser *hp = data_get(self);
+
+  http_parser_init(hp);
+
+  /*
+   * we don't care about trailers in dechunk-only mode,
+   * but if we did we'd set UH_FL_HASTRAILER and clear hp->env
+   */
+  if (0) {
+    rb_funcall(hp->env, id_clear, 0);
+    hp->flags = UH_FL_HASTRAILER;
+  }
+
+  hp->flags |= UH_FL_HASBODY | UH_FL_INBODY | UH_FL_CHUNKED;
+  hp->cs = http_parser_en_ChunkedBody;
+
+  return self;
+}
+
 /**
  * call-seq:
  *    parser.reset => nil
@@ -677,7 +736,8 @@ static VALUE HttpParser_parse(VALUE self)
   }
 
   http_parser_execute(hp, RSTRING_PTR(data), RSTRING_LEN(data));
-  VALIDATE_MAX_LENGTH(hp->offset, HEADER);
+  if (hp->offset > MAX_HEADER_LEN)
+    parser_raise(e413, "HTTP header is too large");
 
   if (hp->cs == http_parser_first_final ||
       hp->cs == http_parser_en_ChunkedBody) {
@@ -690,11 +750,32 @@ static VALUE HttpParser_parse(VALUE self)
   }
 
   if (hp->cs == http_parser_error)
-    parser_error("Invalid HTTP format, parsing fails.");
+    parser_raise(eHttpParserError, "Invalid HTTP format, parsing fails.");
 
   return Qnil;
 }
 
+/**
+ * Document-method: parse
+ * call-seq:
+ *    parser.add_parse(buffer) => env or nil
+ *
+ * adds the contents of +buffer+ to the internal buffer and attempts to
+ * continue parsing.  Returns the +env+ Hash on success or nil if more
+ * data is needed.
+ *
+ * Raises HttpParserError if there are parsing errors.
+ */
+static VALUE HttpParser_add_parse(VALUE self, VALUE buffer)
+{
+  struct http_parser *hp = data_get(self);
+
+  Check_Type(buffer, T_STRING);
+  rb_str_buf_append(hp->buf, buffer);
+
+  return HttpParser_parse(self);
+}
+
 /**
  * Document-method: trailers
  * call-seq:
@@ -803,71 +884,80 @@ static VALUE HttpParser_env(VALUE self)
 
 /**
  * call-seq:
- *    parser.filter_body(buf, data) => nil/data
+ *    parser.filter_body(dst, src) => nil/src
  *
- * Takes a String of +data+, will modify data if dechunking is done.
+ * Takes a String of +src+, will modify data if dechunking is done.
  * Returns +nil+ if there is more data left to process.  Returns
- * +data+ if body processing is complete. When returning +data+,
- * it may modify +data+ so the start of the string points to where
+ * +src+ if body processing is complete. When returning +src+,
+ * it may modify +src+ so the start of the string points to where
  * the body ended so that trailer processing can begin.
  *
  * Raises HttpParserError if there are dechunking errors.
- * Basically this is a glorified memcpy(3) that copies +data+
+ * Basically this is a glorified memcpy(3) that copies +src+
  * into +buf+ while filtering it through the dechunker.
  */
-static VALUE HttpParser_filter_body(VALUE self, VALUE buf, VALUE data)
+static VALUE HttpParser_filter_body(VALUE self, VALUE dst, VALUE src)
 {
   struct http_parser *hp = data_get(self);
-  char *dptr;
-  long dlen;
+  char *srcptr;
+  long srclen;
 
-  dptr = RSTRING_PTR(data);
-  dlen = RSTRING_LEN(data);
+  srcptr = RSTRING_PTR(src);
+  srclen = RSTRING_LEN(src);
 
-  StringValue(buf);
-  rb_str_resize(buf, dlen); /* we can never copy more than dlen bytes */
-  OBJ_TAINT(buf); /* keep weirdo $SAFE users happy */
+  StringValue(dst);
 
   if (HP_FL_TEST(hp, CHUNKED)) {
     if (!chunked_eof(hp)) {
+      rb_str_modify(dst);
+      rb_str_resize(dst, srclen); /* we can never copy more than srclen bytes */
+
       hp->s.dest_offset = 0;
-      hp->cont = buf;
-      hp->buf = data;
-      http_parser_execute(hp, dptr, dlen);
+      hp->cont = dst;
+      hp->buf = src;
+      http_parser_execute(hp, srcptr, srclen);
       if (hp->cs == http_parser_error)
-        parser_error("Invalid HTTP format, parsing fails.");
+        parser_raise(eHttpParserError, "Invalid HTTP format, parsing fails.");
 
       assert(hp->s.dest_offset <= hp->offset &&
              "destination buffer overflow");
-      advance_str(data, hp->offset);
-      rb_str_set_len(buf, hp->s.dest_offset);
+      advance_str(src, hp->offset);
+      rb_str_set_len(dst, hp->s.dest_offset);
 
-      if (RSTRING_LEN(buf) == 0 && chunked_eof(hp)) {
+      if (RSTRING_LEN(dst) == 0 && chunked_eof(hp)) {
         assert(hp->len.chunk == 0 && "chunk at EOF but more to parse");
       } else {
-        data = Qnil;
+        src = Qnil;
       }
     }
   } else {
     /* no need to enter the Ragel machine for unchunked transfers */
     assert(hp->len.content >= 0 && "negative Content-Length");
     if (hp->len.content > 0) {
-      long nr = MIN(dlen, hp->len.content);
-
-      hp->buf = data;
-      memcpy(RSTRING_PTR(buf), dptr, nr);
+      long nr = MIN(srclen, hp->len.content);
+
+      rb_str_modify(dst);
+      rb_str_resize(dst, nr);
+      /*
+       * using rb_str_replace() to avoid memcpy() doesn't help in
+       * most cases because a GC-aware programmer will pass an explicit
+       * buffer to env["rack.input"].read and reuse the buffer in a loop.
+       * This causes copy-on-write behavior to be triggered anyways
+       * when the +src+ buffer is modified (when reading off the socket).
+       */
+      hp->buf = src;
+      memcpy(RSTRING_PTR(dst), srcptr, nr);
       hp->len.content -= nr;
       if (hp->len.content == 0) {
         HP_FL_SET(hp, REQEOF);
         hp->cs = http_parser_first_final;
       }
-      advance_str(data, nr);
-      rb_str_set_len(buf, nr);
-      data = Qnil;
+      advance_str(src, nr);
+      src = Qnil;
     }
   }
   hp->offset = 0; /* for trailer parsing */
-  return data;
+  return src;
 }
 
 #define SET_GLOBAL(var,str) do { \
@@ -883,13 +973,19 @@ void Init_unicorn_http(void)
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
   eHttpParserError =
          rb_define_class_under(mUnicorn, "HttpParserError", rb_eIOError);
+  e413 = rb_define_class_under(mUnicorn, "RequestEntityTooLargeError",
+                               eHttpParserError);
+  e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
+                               eHttpParserError);
 
   init_globals();
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
   rb_define_method(cHttpParser, "reset", HttpParser_reset, 0);
+  rb_define_method(cHttpParser, "dechunk!", HttpParser_dechunk_bang, 0);
   rb_define_method(cHttpParser, "parse", HttpParser_parse, 0);
+  rb_define_method(cHttpParser, "add_parse", HttpParser_add_parse, 1);
   rb_define_method(cHttpParser, "headers", HttpParser_headers, 2);
   rb_define_method(cHttpParser, "trailers", HttpParser_headers, 2);
   rb_define_method(cHttpParser, "filter_body", HttpParser_filter_body, 2);
@@ -924,6 +1020,7 @@ void Init_unicorn_http(void)
   rb_define_singleton_method(cHttpParser, "keepalive_requests=", set_ka_req, 1);
   rb_define_singleton_method(cHttpParser, "trust_x_forwarded=", set_xftrust, 1);
   rb_define_singleton_method(cHttpParser, "trust_x_forwarded?", xftrust, 0);
+  rb_define_singleton_method(cHttpParser, "max_header_len=", set_maxhdrlen, 1);
 
   init_common_fields();
   SET_GLOBAL(g_http_host, "HOST");
@@ -932,6 +1029,7 @@ void Init_unicorn_http(void)
   SET_GLOBAL(g_content_length, "CONTENT_LENGTH");
   SET_GLOBAL(g_http_connection, "CONNECTION");
   id_clear = rb_intern("clear");
+  id_set_backtrace = rb_intern("set_backtrace");
   init_unicorn_httpdate();
 }
 #undef SET_GLOBAL

data/lib/unicorn/configurator.rb

@@ -8,6 +8,8 @@ require 'logger'
 # example configuration files.  An example config file for use with
 # nginx is also available at
 # http://unicorn.bogomips.org/examples/nginx.conf
+#
+# See the link:/TUNING.html document for more information on tuning unicorn.
 class Unicorn::Configurator
   include Unicorn
 
@@ -253,24 +255,50 @@ class Unicorn::Configurator
   #
   # [:tcp_nodelay => true or false]
   #
-  #   Disables Nagle's algorithm on TCP sockets if +true+
+  #   Disables Nagle's algorithm on TCP sockets if +true+.
+  #
+  #   Setting this to +true+ can make streaming responses in Rails 3.1
+  #   appear more quickly at the cost of slightly higher bandwidth usage.
+  #   The effect of this option is most visible if nginx is not used,
+  #   but nginx remains highly recommended with \Unicorn.
   #
   #   This has no effect on UNIX sockets.
   #
-  #   Default: operating system defaults (usually Nagle's algorithm enabled)
+  #   Default: +true+ (Nagle's algorithm disabled) in \Unicorn,
+  #   +true+ in Rainbows!  This defaulted to +false+ in \Unicorn
+  #   3.x
   #
   # [:tcp_nopush => true or false]
   #
   #   Enables/disables TCP_CORK in Linux or TCP_NOPUSH in FreeBSD
   #
-  #   This is enabled by default as of Unicorn 3.4.  This prevents partial
-  #   TCP frames from being sent out and reduces wakeups in nginx if it is
-  #   on a different machine.  Since Unicorn is only designed for applications
-  #   that send the response body quickly without keepalive, sockets will
-  #   always be flushed on close to prevent delays.
+  #   This prevents partial TCP frames from being sent out and reduces
+  #   wakeups in nginx if it is on a different machine.  Since \Unicorn
+  #   is only designed for applications that send the response body
+  #   quickly without keepalive, sockets will always be flushed on close
+  #   to prevent delays.
   #
   #   This has no effect on UNIX sockets.
   #
+  #   Default: +false+
+  #   This defaulted to +true+ in \Unicorn 3.4 - 3.7
+  #
+  # [:ipv6only => true or false]
+  #
+  #   This option makes IPv6-capable TCP listeners IPv6-only and unable
+  #   to receive IPv4 queries on dual-stack systems.  A separate IPv4-only
+  #   listener is required if this is true.
+  #
+  #   This option is only available for Ruby 1.9.2 and later.
+  #
+  #   Enabling this option for the IPv6-only listener and having a
+  #   separate IPv4 listener is recommended if you wish to support IPv6
+  #   on the same TCP port.  Otherwise, the value of \env[\"REMOTE_ADDR\"]
+  #   will appear as an ugly IPv4-mapped-IPv6 address for IPv4 clients
+  #   (e.g ":ffff:10.0.0.1" instead of just "10.0.0.1").
+  #
+  #   Default: Operating-system dependent
+  #
   # [:tries => Integer]
   #
   #   Times to retry binding a socket if it is already in use
@@ -348,7 +376,7 @@ class Unicorn::Configurator
         Integer === value or
           raise ArgumentError, "not an integer: #{key}=#{value.inspect}"
       end
-      [ :tcp_nodelay, :tcp_nopush ].each do |key|
+      [ :tcp_nodelay, :tcp_nopush, :ipv6only ].each do |key|
         (value = options[key]).nil? and next
         TrueClass === value || FalseClass === value or
           raise ArgumentError, "not boolean: #{key}=#{value.inspect}"

data/lib/unicorn/const.rb

@@ -8,8 +8,8 @@
 # improve things much compared to constants.
 module Unicorn::Const
 
-  # The current version of Unicorn, currently 3.4.0
-  UNICORN_VERSION = "3.4.0"
+  # The current version of Unicorn, currently 4.0.0
+  UNICORN_VERSION = "4.0.0"
 
   # default TCP listen host address (0.0.0.0, all interfaces)
   DEFAULT_HOST = "0.0.0.0"
@@ -25,12 +25,14 @@ module Unicorn::Const
 
   # Maximum request body size before it is moved out of memory and into a
   # temporary file for reading (112 kilobytes).  This is the default
-  # value of of client_body_buffer_size.
+  # value of client_body_buffer_size.
   MAX_BODY = 1024 * 112
 
   # :stopdoc:
   # common errors we'll send back
   ERROR_400_RESPONSE = "HTTP/1.1 400 Bad Request\r\n\r\n"
+  ERROR_414_RESPONSE = "HTTP/1.1 414 Request-URI Too Long\r\n\r\n"
+  ERROR_413_RESPONSE = "HTTP/1.1 413 Request Entity Too Large\r\n\r\n"
   ERROR_500_RESPONSE = "HTTP/1.1 500 Internal Server Error\r\n\r\n"
   EXPECT_100_RESPONSE = "HTTP/1.1 100 Continue\r\n\r\n"
 

data/lib/unicorn/http_request.rb

@@ -68,9 +68,7 @@ class Unicorn::HttpParser
     if parse.nil?
       # Parser is not done, queue up more data to read and continue parsing
       # an Exception thrown from the parser will throw us out of the loop
-      begin
-        buf << socket.kgio_read!(16384)
-      end while parse.nil?
+      false until add_parse(socket.kgio_read!(16384))
     end
     e[RACK_INPUT] = 0 == content_length ?
                     NULL_IO : @@input_class.new(socket, self)