unicorn 0.9.2 → 0.90.0

data/.gitignore

@@ -8,6 +8,7 @@
 /local.mk
 /test/install-*
 ext/unicorn_http/Makefile
+ext/unicorn_http/unicorn_http.c
 log/
 pkg/
 /vendor

data/CHANGELOG

@@ -1,6 +1,9 @@
+v0.90.0 - switch chunking+trailer handling to Ragel, v0.8.4 fixes
 v0.9.2 - Ruby 1.9.2 preview1 compatibility
 v0.9.1 - FD_CLOEXEC portability fix (v0.8.2 port)
 v0.9.0 - bodies: "Transfer-Encoding: chunked", rewindable streaming
+v0.8.4 - pass through unknown HTTP status codes
+v0.8.3 - Ruby 1.9.2 preview1 compatibility
 v0.8.2 - socket handling bugfixes and usability tweaks
 v0.8.1 - safer timeout handling, more consistent reload behavior
 v0.8.0 - enforce Rack dependency, minor performance improvements and fixes

data/GNUmakefile

@@ -28,13 +28,13 @@ T_r_log := $(subst .r,$(log_suffix),$(T_r))
 test_prefix = $(CURDIR)/test/install-$(RUBY_VERSION)
 
 ext := ext/unicorn_http
-c_files := $(addprefix $(ext)/,ext_help.h unicorn_http.c unicorn_http.h)
+c_files := $(ext)/unicorn_http.c $(wildcard $(ext)/*.h)
 rl_files := $(addprefix $(ext)/,unicorn_http.rl unicorn_http_common.rl)
 rb_files := $(shell grep '^\(bin\|lib\)' Manifest)
 inst_deps := $(c_files) $(rb_files)
 
-ragel: $(ext)/unicorn_http.h
-$(ext)/unicorn_http.h: $(rl_files)
+ragel: $(ext)/unicorn_http.c
+$(ext)/unicorn_http.c: $(rl_files)
 	cd $(@D) && $(ragel) unicorn_http.rl -C $(RLFLAGS) -o $(@F)
 	$(ruby) -i -p -e '$$_.gsub!(%r{[ \t]*$$},"")' $@
 $(ext)/Makefile: $(ext)/extconf.rb $(c_files)
@@ -103,14 +103,14 @@ $(T): export RUBYLIB := $(test_prefix):$(test_prefix)/lib:$(RUBYLIB)
 $(T): $(test_prefix)/.stamp
 	$(run_test)
 
-install: $(bins)
+install: $(bins) $(ext)/unicorn_http.c
 	$(prep_setup_rb)
 	$(RM) -r .install-tmp
 	mkdir .install-tmp
-	cp -p $^ .install-tmp
+	cp -p bin/* .install-tmp
 	$(ruby) setup.rb all
 	$(RM) $^
-	mv $(addprefix .install-tmp/,$(^F)) bin/
+	mv .install-tmp/* bin/
 	$(RM) -r .install-tmp
 	$(prep_setup_rb)
 
@@ -124,7 +124,7 @@ clean:
 	$(RM) -r $(test_prefix)
 
 Manifest:
-	git ls-files > $@+
+	(git ls-files && echo $(ext)/unicorn_http.c) | LC_ALL=C sort > $@+
 	cmp $@+ $@ || mv $@+ $@
 	$(RM) -f $@+
 

data/Manifest

@@ -18,10 +18,12 @@ bin/unicorn_rails
 examples/echo.ru
 examples/git.ru
 examples/init.sh
+ext/unicorn_http/c_util.h
+ext/unicorn_http/common_field_optimization.h
 ext/unicorn_http/ext_help.h
 ext/unicorn_http/extconf.rb
+ext/unicorn_http/global_variables.h
 ext/unicorn_http/unicorn_http.c
-ext/unicorn_http/unicorn_http.h
 ext/unicorn_http/unicorn_http.rl
 ext/unicorn_http/unicorn_http_common.rl
 lib/unicorn.rb
@@ -30,7 +32,6 @@ lib/unicorn/app/inetd.rb
 lib/unicorn/app/old_rails.rb
 lib/unicorn/app/old_rails/static.rb
 lib/unicorn/cgi_wrapper.rb
-lib/unicorn/chunked_reader.rb
 lib/unicorn/configurator.rb
 lib/unicorn/const.rb
 lib/unicorn/http_request.rb
@@ -38,16 +39,12 @@ lib/unicorn/http_response.rb
 lib/unicorn/launcher.rb
 lib/unicorn/socket_helper.rb
 lib/unicorn/tee_input.rb
-lib/unicorn/trailer_parser.rb
 lib/unicorn/util.rb
 local.mk.sample
 setup.rb
 test/aggregate.rb
 test/benchmark/README
-test/benchmark/big_request.rb
 test/benchmark/dd.ru
-test/benchmark/request.rb
-test/benchmark/response.rb
 test/exec/README
 test/exec/test_exec.rb
 test/rails/app-1.2.3/.gitignore
@@ -110,31 +107,31 @@ test/rails/app-2.2.2/db/.gitignore
 test/rails/app-2.2.2/log/.gitignore
 test/rails/app-2.2.2/public/404.html
 test/rails/app-2.2.2/public/500.html
-test/rails/app-2.3.2.1/.gitignore
-test/rails/app-2.3.2.1/Rakefile
-test/rails/app-2.3.2.1/app/controllers/application_controller.rb
-test/rails/app-2.3.2.1/app/controllers/foo_controller.rb
-test/rails/app-2.3.2.1/app/helpers/application_helper.rb
-test/rails/app-2.3.2.1/config/boot.rb
-test/rails/app-2.3.2.1/config/database.yml
-test/rails/app-2.3.2.1/config/environment.rb
-test/rails/app-2.3.2.1/config/environments/development.rb
-test/rails/app-2.3.2.1/config/environments/production.rb
-test/rails/app-2.3.2.1/config/routes.rb
-test/rails/app-2.3.2.1/db/.gitignore
-test/rails/app-2.3.2.1/log/.gitignore
-test/rails/app-2.3.2.1/public/404.html
-test/rails/app-2.3.2.1/public/500.html
+test/rails/app-2.3.3.1/.gitignore
+test/rails/app-2.3.3.1/Rakefile
+test/rails/app-2.3.3.1/app/controllers/application_controller.rb
+test/rails/app-2.3.3.1/app/controllers/foo_controller.rb
+test/rails/app-2.3.3.1/app/helpers/application_helper.rb
+test/rails/app-2.3.3.1/config/boot.rb
+test/rails/app-2.3.3.1/config/database.yml
+test/rails/app-2.3.3.1/config/environment.rb
+test/rails/app-2.3.3.1/config/environments/development.rb
+test/rails/app-2.3.3.1/config/environments/production.rb
+test/rails/app-2.3.3.1/config/routes.rb
+test/rails/app-2.3.3.1/db/.gitignore
+test/rails/app-2.3.3.1/log/.gitignore
+test/rails/app-2.3.3.1/public/404.html
+test/rails/app-2.3.3.1/public/500.html
 test/rails/test_rails.rb
 test/test_helper.rb
-test/unit/test_chunked_reader.rb
 test/unit/test_configurator.rb
 test/unit/test_http_parser.rb
+test/unit/test_http_parser_ng.rb
 test/unit/test_request.rb
 test/unit/test_response.rb
 test/unit/test_server.rb
 test/unit/test_signals.rb
 test/unit/test_socket_helper.rb
-test/unit/test_trailer_parser.rb
+test/unit/test_tee_input.rb
 test/unit/test_upload.rb
 test/unit/test_util.rb

data/README

@@ -3,16 +3,14 @@
 == Features
 
 * Designed for Rack, Unix, fast clients, and ease-of-debugging.  We
-  cut out all things that are better-supported by nginx or Rack.
+  cut out everything that is better supported by the operating system,
+  nginx or Rack.
 
-* Mostly written in Ruby, only the HTTP parser (stolen and trimmed
-  down from Mongrel) is written in C.  Unicorn is compatible with
-  both Ruby 1.8 and 1.9.  A pure-Ruby (but still Unix-only) version
-  is planned.
+* Compatible with both Ruby 1.8 and 1.9, Rubinius support is planned.
 
 * Process management: Unicorn will reap and restart workers that
   die from broken apps.  There is no need to manage multiple processes
-  or ports yourself.  Unicorn can spawn and manage any fixed number of
+  or ports yourself.  Unicorn can spawn and manage any number of
   worker processes you choose to scale to your backend.
 
 * Load balancing is done entirely by the operating system kernel.
@@ -20,7 +18,7 @@
 
 * Does not care if your application is thread-safe or not, workers
   all run within their own isolated address space and only serve one
-  client at a time.
+  client at a time for maximum robustness.
 
 * Supports all Rack applications, along with pre-Rack versions of
   Ruby on Rails via a Rack wrapper.
@@ -31,10 +29,9 @@
   Unicorn also takes steps to ensure multi-line log entries from one
   request all stay within the same file.
 
-* nginx-style binary re-execution without losing connections.
+* nginx-style binary upgrades without losing connections.
   You can upgrade Unicorn, your entire application, libraries
-  and even your Ruby interpreter as long as Unicorn is
-  installed in the same path.
+  and even your Ruby interpreter without dropping clients.
 
 * before_fork and after_fork hooks in case your application
   has special needs when dealing with forked processes.  These
@@ -48,11 +45,17 @@
   each worker process can also bind to a private port via the
   after_fork hook for easy debugging.
 
+* Simple and easy Ruby DSL for configuration.
+
+* Decodes chunked transfers on-the-fly, thus allowing upload progress
+  notification to be implemented as well as being able to tunnel
+  arbitrary stream-based protocols over HTTP.
+
 == Versions
 
-{Stable v0.8.2}[http://git.bogomips.org/cgit/unicorn.git/tag/?id=v0.8.2]
+{Current v0.90.0}[http://git.bogomips.org/cgit/unicorn.git/tag/?id=v0.90.0]
 
-{Development v0.9.1}[http://git.bogomips.org/cgit/unicorn.git/tag/?id=v0.9.1]
+{Old Stable v0.8.4}[http://git.bogomips.org/cgit/unicorn.git/tag/?id=v0.8.4]
 
 == License
 
@@ -77,7 +80,7 @@ http://rubyforge.org/frs/?group_id=1306
 
 You may also install it via Rubygems on Rubyforge:
 
-  gem install unicorn -v0.8.2
+  gem install unicorn
 
 You can get the latest source via git from the following locations
 (these versions may not be stable):

data/TODO

@@ -1,9 +1,11 @@
-* integration tests with nginx including bad client handling
+* security audit, fuzz testing
 
 * manpages (why do so few Ruby executables come with proper manpages?)
 
-* code cleanups (launchers)
+* code cleanups
 
-* Pure Ruby HTTP parser
+* launchers - add stop/start/kill/upgrade commands
+
+* Rainbows!
 
 * Rubinius support

data/bin/unicorn

@@ -120,10 +120,10 @@ require 'pp' if $DEBUG
 app = lambda do ||
   # require Rack as late as possible in case $LOAD_PATH is modified
   # in config.ru or command-line
-  require 'rack'
   inner_app = case config
   when /\.ru$/
     raw = File.open(config, "rb") { |fp| fp.sysread(fp.stat.size) }
+    raw.sub!(/^__END__\n.*/, '')
     eval("Rack::Builder.new {(#{raw}\n)}.to_app", nil, config)
   else
     require config

data/bin/unicorn_rails

@@ -140,7 +140,6 @@ app = lambda do ||
     end
 
     if old_rails
-      require 'rack'
       require 'unicorn/app/old_rails'
       Unicorn::App::OldRails.new
     else
@@ -148,6 +147,7 @@ app = lambda do ||
     end
   when /\.ru$/
     raw = File.open(config, "rb") { |fp| fp.sysread(fp.stat.size) }
+    raw.sub!(/^__END__\n.*/, '')
     eval("Rack::Builder.new {(#{raw}\n)}.to_app", nil, config)
   else
     require config

data/ext/unicorn_http/c_util.h

@@ -0,0 +1,107 @@
+/*
+ * Generic C functions and macros go here, there are no dependencies
+ * on Unicorn internal structures or the Ruby C API in here.
+ */
+
+#ifndef UH_util_h
+#define UH_util_h
+
+#include <unistd.h>
+#include <assert.h>
+
+#define MIN(a,b) (a < b ? a : b)
+#define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
+
+#ifndef SIZEOF_OFF_T
+#  define SIZEOF_OFF_T 4
+#  warning SIZEOF_OFF_T not defined, guessing 4.  Did you run extconf.rb?
+#endif
+
+#if SIZEOF_OFF_T == 4
+#  define UH_OFF_T_MAX 0x7fffffff
+#elif SIZEOF_OFF_T == 8
+#  if SIZEOF_LONG == 4
+#    define UH_OFF_T_MAX 0x7fffffffffffffffLL
+#  else
+#    define UH_OFF_T_MAX 0x7fffffffffffffff
+#  endif
+#else
+#  error off_t size unknown for this platform!
+#endif
+
+/*
+ * capitalizes all lower-case ASCII characters and converts dashes
+ * to underscores for HTTP headers.  Locale-agnostic.
+ */
+static void snake_upcase_char(char *c)
+{
+  if (*c >= 'a' && *c <= 'z')
+    *c &= ~0x20;
+  else if (*c == '-')
+    *c = '_';
+}
+
+/* Downcases a single ASCII character.  Locale-agnostic. */
+static void downcase_char(char *c)
+{
+  if (*c >= 'A' && *c <= 'Z')
+    *c |= 0x20;
+}
+
+static int hexchar2int(int xdigit)
+{
+  if (xdigit >= 'A' && xdigit <= 'F')
+    return xdigit - 'A' + 10;
+  if (xdigit >= 'a' && xdigit <= 'f')
+    return xdigit - 'a' + 10;
+
+  /* Ragel already does runtime range checking for us in Unicorn: */
+  assert(xdigit >= '0' && xdigit <= '9');
+
+  return xdigit - '0';
+}
+
+/*
+ * multiplies +i+ by +base+ and increments the result by the parsed
+ * integer value of +xdigit+.  +xdigit+ is a character byte
+ * representing a number the range of 0..(base-1)
+ * returns the new value of +i+ on success
+ * returns -1 on errors (including overflow)
+ */
+static off_t step_incr(off_t i, int xdigit, const int base)
+{
+  static const off_t max = UH_OFF_T_MAX;
+  const off_t next_max = (max - (max % base)) / base;
+  off_t offset = hexchar2int(xdigit);
+
+  if (offset > (base - 1))
+    return -1;
+  if (i > next_max)
+    return -1;
+  i *= base;
+
+  if ((offset > (base - 1)) || ((max - i) < offset))
+    return -1;
+
+  return i + offset;
+}
+
+/*
+ * parses a non-negative length according to base-10 and
+ * returns it as an off_t value.  Returns -1 on errors
+ * (including overflow).
+ */
+static off_t parse_length(const char *value, size_t length)
+{
+  off_t rv;
+
+  for (rv = 0; length-- && rv >= 0; ++value)
+    rv = step_incr(rv, *value, 10);
+
+  return rv;
+}
+
+#define CONST_MEM_EQ(const_p, buf, len) \
+  ((sizeof(const_p) - 1) == len && !memcmp(const_p, buf, sizeof(const_p) - 1))
+
+#endif /* UH_util_h */

data/ext/unicorn_http/common_field_optimization.h

@@ -0,0 +1,110 @@
+#ifndef common_field_optimization
+#define common_field_optimization
+#include "ruby.h"
+#include "c_util.h"
+
+struct common_field {
+  const signed long len;
+  const char *name;
+  VALUE value;
+};
+
+/*
+ * A list of common HTTP headers we expect to receive.
+ * This allows us to avoid repeatedly creating identical string
+ * objects to be used with rb_hash_aset().
+ */
+static struct common_field common_http_fields[] = {
+# define f(N) { (sizeof(N) - 1), N, Qnil }
+  f("ACCEPT"),
+  f("ACCEPT_CHARSET"),
+  f("ACCEPT_ENCODING"),
+  f("ACCEPT_LANGUAGE"),
+  f("ALLOW"),
+  f("AUTHORIZATION"),
+  f("CACHE_CONTROL"),
+  f("CONNECTION"),
+  f("CONTENT_ENCODING"),
+  f("CONTENT_LENGTH"),
+  f("CONTENT_TYPE"),
+  f("COOKIE"),
+  f("DATE"),
+  f("EXPECT"),
+  f("FROM"),
+  f("HOST"),
+  f("IF_MATCH"),
+  f("IF_MODIFIED_SINCE"),
+  f("IF_NONE_MATCH"),
+  f("IF_RANGE"),
+  f("IF_UNMODIFIED_SINCE"),
+  f("KEEP_ALIVE"), /* Firefox sends this */
+  f("MAX_FORWARDS"),
+  f("PRAGMA"),
+  f("PROXY_AUTHORIZATION"),
+  f("RANGE"),
+  f("REFERER"),
+  f("TE"),
+  f("TRAILER"),
+  f("TRANSFER_ENCODING"),
+  f("UPGRADE"),
+  f("USER_AGENT"),
+  f("VIA"),
+  f("X_FORWARDED_FOR"), /* common for proxies */
+  f("X_FORWARDED_PROTO"), /* common for proxies */
+  f("X_REAL_IP"), /* common for proxies */
+  f("WARNING")
+# undef f
+};
+
+#define HTTP_PREFIX "HTTP_"
+#define HTTP_PREFIX_LEN (sizeof(HTTP_PREFIX) - 1)
+
+/* this function is not performance-critical, called only at load time */
+static void init_common_fields(void)
+{
+  int i;
+  struct common_field *cf = common_http_fields;
+  char tmp[64];
+  memcpy(tmp, HTTP_PREFIX, HTTP_PREFIX_LEN);
+
+  for(i = 0; i < ARRAY_SIZE(common_http_fields); cf++, i++) {
+    /* Rack doesn't like certain headers prefixed with "HTTP_" */
+    if (!strcmp("CONTENT_LENGTH", cf->name) ||
+        !strcmp("CONTENT_TYPE", cf->name)) {
+      cf->value = rb_str_new(cf->name, cf->len);
+    } else {
+      memcpy(tmp + HTTP_PREFIX_LEN, cf->name, cf->len + 1);
+      cf->value = rb_str_new(tmp, HTTP_PREFIX_LEN + cf->len);
+    }
+    cf->value = rb_obj_freeze(cf->value);
+    rb_global_variable(&cf->value);
+  }
+}
+
+/* this function is called for every header set */
+static VALUE find_common_field(const char *field, size_t flen)
+{
+  int i;
+  struct common_field *cf = common_http_fields;
+
+  for(i = 0; i < ARRAY_SIZE(common_http_fields); i++, cf++) {
+    if (cf->len == flen && !memcmp(cf->name, field, flen))
+      return cf->value;
+  }
+  return Qnil;
+}
+
+/*
+ * We got a strange header that we don't have a memoized value for.
+ * Fallback to creating a new string to use as a hash key.
+ */
+static VALUE uncommon_field(const char *field, size_t flen)
+{
+  VALUE f = rb_str_new(NULL, HTTP_PREFIX_LEN + flen);
+  memcpy(RSTRING_PTR(f), HTTP_PREFIX, HTTP_PREFIX_LEN);
+  memcpy(RSTRING_PTR(f) + HTTP_PREFIX_LEN, field, flen);
+  assert(*(RSTRING_PTR(f) + RSTRING_LEN(f)) == '\0'); /* paranoia */
+  return rb_obj_freeze(f);
+}
+
+#endif /* common_field_optimization_h */