yahns 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e34f2f1a085c816144583ee7e457ae3f3225b1cc
-  data.tar.gz: 4e8ade4e659196c6d3ac44f3f5985b07c649b10c
+  metadata.gz: c03354f508bb8d5ece9deaac06b265c9f97c7254
+  data.tar.gz: e0bcd2f85d316a6ff76b9e7f38b9449f088e59aa
 SHA512:
-  metadata.gz: 66a99f3af39793a249cb5efb09c11da88d883548568a9af6c6f2750befd16a6441e42f434012215adb9294500730aa5b0df8d51d2db6a8ad7efe47cd729e23aa
-  data.tar.gz: e1248978eaa6256c779d780d1f0ae4e01652c9435bc124d20d1f6c23b0cfc158357bd8e4df836c77b8935818d0d4825400e0b0928bfead879bad71349654ce6b
+  metadata.gz: d1cf5fae66ee58b4a5435d2d760c6b4283a327e58919c67bcfc7a6ec4805b3a80747e47a9565dab2042bebcfd9ec210ee7f6665397e863ab7cd03cddb9b34493
+  data.tar.gz: 761d334654d9b44ba58f96fbe78d087a8e9c323ce55ef5d1597e6681d5d0076d34f41918858aca66ab3056ce2b52f524d5a2d5ba4f70fc60916f815c1213710b

data/GIT-VERSION-GEN

@@ -4,7 +4,7 @@
 CONSTANT = "Yahns::VERSION"
 RVF = "lib/yahns/version.rb"
 GVF = "GIT-VERSION-FILE"
-DEF_VER = "v0.0.0"
+DEF_VER = "v0.0.1"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/GNUmakefile

@@ -12,11 +12,11 @@ all:: test
 test_units := $(wildcard test/test_*.rb)
 test: $(test_units)
 $(test_units):
-	$(RUBY) -w -I $(lib) $@ -v
+	$(RUBY) -I $(lib) $@ -v
 
 test-mt: export N = $(shell nproc 2>/dev/null || echo 4)
 test-mt:
-	$(RUBY) -w -I $(lib) $(addprefix -r./,$(test_units)) -eexit --
+	$(RUBY) -I $(lib) $(addprefix -r./,$(test_units)) -eTime.now --
 
 check-warnings:
 	@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \

data/lib/yahns/acceptor.rb

@@ -2,8 +2,8 @@
 # License: GPLv3 or later (see COPYING for details)
 module Yahns::Acceptor # :nodoc:
   def spawn_acceptor(logger, client_class, queue)
-    accept_flags = Kgio::SOCK_NONBLOCK | Kgio::SOCK_CLOEXEC
     Thread.new do
+      accept_flags = Kgio::SOCK_NONBLOCK | Kgio::SOCK_CLOEXEC
       Thread.current.abort_on_exception = true
       qev_flags = client_class.superclass::QEV_FLAGS
       begin
@@ -21,8 +21,10 @@ module Yahns::Acceptor # :nodoc:
         queue.fdmap.desperate_expire_for(self, 5)
         sleep 1 # let other threads do some work
       rescue => e
-        Yahns::Log.exception(logger, "accept loop error", e) unless closed?
-      end until closed?
+        # sleep since this check is racy (and uncommon)
+        break if closed? || (sleep(0.01) && closed?)
+        Yahns::Log.exception(logger, "accept loop", e)
+      end while true
     end
   end
 end

data/lib/yahns/cap_input.rb

@@ -0,0 +1,22 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+
+# This is used as the @input/env["rack.input"] when
+# input_buffering == true or :lazy
+class Yahns::CapInput < Yahns::TmpIO # :nodoc:
+  attr_writer :bytes_left
+
+  def self.new(limit)
+    rv = super()
+    rv.bytes_left = limit
+    rv
+  end
+
+  def write(buf)
+    if (@bytes_left -= buf.size) < 0
+      raise Unicorn::RequestEntityTooLargeError, "chunked body too big", []
+    end
+    super(buf)
+  end
+end

data/lib/yahns/config.rb

@@ -285,7 +285,6 @@ class Yahns::Config # :nodoc:
   {
     # config name, minimum value
     client_body_buffer_size: 1,
-    client_max_body_size: 0,
     client_header_buffer_size: 1,
     client_max_header_size: 1,
     client_timeout: 0,
@@ -298,6 +297,12 @@ class Yahns::Config # :nodoc:
     )
   end
 
+  def client_max_body_size(val)
+    var = _check_in_block(:app, :client_max_body_size)
+    val = _check_int(var, val, 0) if val != nil
+    @block.ctx.__send__("#{var}=", val)
+  end
+
   def input_buffering(val)
     var = _check_in_block(:app, :input_buffering)
     ok = [ :lazy, true, false ]
@@ -312,8 +317,8 @@ class Yahns::Config # :nodoc:
     if String === val
       # we've already bound working_directory by the time we get here
       val = File.open(File.expand_path(val), "a")
+      val.close_on_exec = val.sync = true
       val.binmode
-      val.sync = true
     else
       rt = %w(puts write flush).map(&:to_sym) # match Rack::Lint
       rt.all? { |m| val.respond_to?(m) } or raise ArgumentError,
@@ -330,7 +335,7 @@ class Yahns::Config # :nodoc:
         @set[key] = path = "/dev/null"
       end
       File.open(path, 'a') { |fp| io.reopen(fp) } if String === path
-      io.sync = true
+      io.close_on_exec = io.sync = true
     end
 
     [ :logger, :pid, :worker_processes ].each do |var|
@@ -338,7 +343,6 @@ class Yahns::Config # :nodoc:
       server.__send__("#{var}=", val) if val != :unset
     end
     queue(:default) if @qeggs.empty?
-    @qeggs.each_value { |qegg| qegg.logger ||= server.logger }
     @app_ctx.each { |app| app.logger ||= server.logger }
   end
 end

data/lib/yahns/daemon.rb

@@ -25,6 +25,7 @@ module Yahns::Daemon # :nodoc:
       # We cannot use Yahns::Sigevent (eventfd) here because we need
       # to detect EOF on unexpected death, not just read/write
       rd, wr = IO.pipe
+      rd.close_on_exec = wr.close_on_exec = true
       grandparent = $$
       if fork
         wr.close # grandparent does not write

data/lib/yahns/fdmap.rb

@@ -54,14 +54,6 @@ class Yahns::Fdmap # :nodoc:
     @fdmap_mtx.synchronize { @count -= 1 }
   end
 
-  def delete(io) # use with rack.hijack (via yahns)
-    fd = io.fileno
-    @fdmap_mtx.synchronize do
-      @fdmap_ary[fd] = nil
-      @count -= 1
-    end
-  end
-
   # expire a bunch of idle clients and register the current one
   # We should not be calling this too frequently, it is expensive
   # This is called while @fdmap_mtx is held

data/lib/yahns/http_client.rb

@@ -33,9 +33,9 @@ class Yahns::HttpClient < Kgio::Socket # :nodoc:
     case rv = @state.wbuf_flush(self)
     when :wait_writable, :wait_readable
       return rv # tell epoll/kqueue to wait on this more
-    when :delete # :delete on hijack
-      @state = :delete
-      return :delete
+    when :ignore # :ignore on hijack
+      @state = :ignore
+      return :ignore
     when Yahns::StreamFile
       @state = rv # continue looping
     when true, false # done
@@ -45,9 +45,17 @@ class Yahns::HttpClient < Kgio::Socket # :nodoc:
     end while true
   end
 
+  # used only with "input_buffering true"
   def mkinput_preread
+    k = self.class
+    len = @hs.content_length
+    mbs = k.client_max_body_size
+    if mbs && len && len > mbs
+      raise Unicorn::RequestEntityTooLargeError,
+            "Content-Length:#{len} too large (>#{mbs})", []
+    end
     @state = :body
-    @input = self.class.tmpio_for(@hs.content_length)
+    @input = k.tmpio_for(len)
     rbuf = Thread.current[:yahns_rbuf]
     @hs.filter_body(rbuf, @hs.buf)
     @input.write(rbuf)
@@ -151,9 +159,9 @@ class Yahns::HttpClient < Kgio::Socket # :nodoc:
 
     # run the rack app
     response = k.app.call(env.merge!(k.app_defaults))
-    return :delete if env.include?(RACK_HIJACK_IO)
+    return :ignore if env.include?(RACK_HIJACK_IO)
 
-    # this returns :wait_readable, :wait_writable, :delete, or nil:
+    # this returns :wait_readable, :wait_writable, :ignore, or nil:
     http_response_write(*response)
   end
 

data/lib/yahns/http_context.rb

@@ -24,7 +24,7 @@ module Yahns::HttpContext # :nodoc:
     @check_client_connection = false
     @client_body_buffer_size = 112 * 1024
     @client_header_buffer_size = 4000
-    @client_max_body_size = 1024 * 1024
+    @client_max_body_size = 1024 * 1024 # nil => infinity
     @input_buffering = true
     @output_buffering = true
     @persistent_connections = true
@@ -34,7 +34,19 @@ module Yahns::HttpContext # :nodoc:
 
   # call this after forking
   def after_fork_init
-    @app = @yahns_rack.app_after_fork
+    @app = __wrap_app(@yahns_rack.app_after_fork)
+  end
+
+  def __wrap_app(app)
+    # input_buffering == false is handled in http_client
+    return app if @client_max_body_size == nil
+
+    require 'yahns/cap_input'
+    return app if @input_buffering == true
+
+    # @input_buffering == false/:lazy
+    require 'yahns/max_body'
+    Yahns::MaxBody.new(app, @client_max_body_size)
   end
 
   # call this immediately after successful accept()/accept4()
@@ -59,7 +71,11 @@ module Yahns::HttpContext # :nodoc:
   end
 
   def tmpio_for(len)
-    len && len <= @client_body_buffer_size ?
-      StringIO.new("") : Yahns::TmpIO.new
+    if len # Content-Length given
+      len <= @client_body_buffer_size ? StringIO.new("") : Yahns::TmpIO.new
+    else # chunked, unknown length
+      mbs = @client_max_body_size
+      mbs ? Yahns::CapInput.new(mbs) : Yahns::TmpIO.new
+    end
   end
 end

data/lib/yahns/http_response.rb

@@ -52,8 +52,8 @@ module Yahns::HttpResponse # :nodoc:
     case rv # trysendfile return value
     when nil
       case alive
-      when :delete
-        @state = :delete
+      when :ignore
+        @state = :ignore
       when true, false
         http_response_done(alive)
       end
@@ -140,7 +140,7 @@ module Yahns::HttpResponse # :nodoc:
 
     if hijack
       hijack.call(self)
-      return :delete # trigger EPOLL_CTL_DEL
+      return :ignore
     end
 
     if body.respond_to?(:to_path)

data/lib/yahns/max_body.rb

@@ -0,0 +1,60 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+
+# Middleware used to enforce client_max_body_size for TeeInput users.
+#
+# There is no need to configure this middleware manually, it will
+# automatically be configured for you based on the client_max_body_size
+# setting.
+#
+# For more fine-grained control, you may also define it per-endpoint in
+# your Rack config.ru like this:
+#
+#        map "/limit_1M" do
+#          use Yahns::MaxBody, 1024*1024
+#          run MyApp
+#        end
+#        map "/limit_10M" do
+#          use Yahns::MaxBody, 1024*1024*10
+#          run MyApp
+#        end
+class Yahns::MaxBody # :nodoc:
+  # This is automatically called when used with Rack::Builder#use
+  # See Yahns::MaxBody
+  def initialize(app, limit)
+    Integer === limit or raise ArgumentError, "limit not an Integer"
+    @app = app
+    @limit = limit
+  end
+
+  RACK_INPUT = "rack.input".freeze # :nodoc:
+  CONTENT_LENGTH = "CONTENT_LENGTH" # :nodoc:
+  HTTP_TRANSFER_ENCODING = "HTTP_TRANSFER_ENCODING" # :nodoc:
+
+  # our main Rack middleware endpoint
+  def call(env) # :nodoc:
+    catch(:yahns_EFBIG) do
+      len = env[CONTENT_LENGTH]
+      if len && len.to_i > @limit
+        return err
+      elsif /\Achunked\z/i =~ env[HTTP_TRANSFER_ENCODING]
+        limit_input!(env)
+      end
+      @app.call(env)
+    end || err
+  end
+
+  # Rack response returned when there's an error
+  def err # :nodoc:
+    [ 413, { 'Content-Length' => '0', 'Content-Type' => 'text/plain' }, [] ]
+  end
+
+  def limit_input!(env) # :nodoc:
+    input = env[RACK_INPUT]
+    klass = input.respond_to?(:rewind) ? RewindableWrapper : Wrapper
+    env[RACK_INPUT] = klass.new(input, @limit)
+  end
+end
+require_relative 'max_body/wrapper'
+require_relative 'max_body/rewindable_wrapper'

data/lib/yahns/max_body/rewindable_wrapper.rb

@@ -0,0 +1,19 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+class Yahns::MaxBody::RewindableWrapper < Yahns::MaxBody::Wrapper # :nodoc:
+  def initialize(rack_input, limit)
+    @orig_limit = limit
+    super
+  end
+
+  def rewind
+    @limit = @orig_limit
+    @rbuf = ''
+    @input.rewind
+  end
+
+  def size
+    @input.size
+  end
+end

data/lib/yahns/max_body/wrapper.rb

@@ -0,0 +1,71 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+#
+# This is only used for chunked request bodies, which are rare
+class Yahns::MaxBody::Wrapper # :nodoc:
+  def initialize(rack_input, limit)
+    @input, @limit, @rbuf = rack_input, limit, ''
+  end
+
+  def each
+    while line = gets
+      yield line
+    end
+  end
+
+  # chunked encoding means this method behaves more like readpartial,
+  # since Rack does not support a method named "readpartial"
+  def read(length = nil, rv = '')
+    if length
+      if length <= @rbuf.size
+        length < 0 and raise ArgumentError, "negative length #{length} given"
+        rv.replace(@rbuf.slice!(0, length))
+      elsif @rbuf.empty?
+        checked_read(length, rv) or return
+      else
+        rv.replace(@rbuf.slice!(0, @rbuf.size))
+      end
+      rv.empty? && length != 0 ? nil : rv
+    else
+      rv.replace(read_all)
+    end
+  end
+
+  def gets
+    sep = $/
+    if sep.nil?
+      rv = read_all
+      return rv.empty? ? nil : rv
+    end
+    re = /\A(.*?#{Regexp.escape(sep)})/
+
+    begin
+      @rbuf.sub!(re, '') and return $1
+
+      if tmp = checked_read(16384)
+        @rbuf << tmp
+      elsif @rbuf.empty? # EOF
+        return nil
+      else # EOF, return whatever is left
+        return @rbuf.slice!(0, @rbuf.size)
+      end
+    end while true
+  end
+
+  def checked_read(length = 16384, buf = '')
+    if @input.read(length, buf)
+      throw :yahns_EFBIG if ((@limit -= buf.size) < 0)
+      return buf
+    end
+  end
+
+  def read_all
+    rv = @rbuf.slice!(0, @rbuf.size)
+    tmp = ''
+    while checked_read(16384, tmp)
+      rv << tmp
+    end
+    rv
+  end
+end

data/lib/yahns/queue_egg.rb

@@ -4,20 +4,17 @@
 # this represents a Yahns::Queue before its vivified.  This only
 # lives in the parent process and should be clobbered after qc_vivify
 class Yahns::QueueEgg # :nodoc:
-  attr_writer :max_events, :worker_threads
-  attr_accessor :logger
+  attr_accessor :max_events, :worker_threads
 
   def initialize
     @max_events = 1 # 1 is good if worker_threads > 1
     @worker_threads = 7 # any default is wrong for most apps...
-    @logger = nil
   end
 
   # only call after forking
-  def qc_vivify(fdmap)
+  def vivify(fdmap)
     queue = Yahns::Queue.new
     queue.fdmap = fdmap
-    queue.spawn_worker_threads(@logger, @worker_threads, @max_events)
     queue
   end
 end

data/lib/yahns/queue_epoll.rb

@@ -22,36 +22,34 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
   end
 
   # returns an array of infinitely running threads
-  def spawn_worker_threads(logger, worker_threads, max_events)
-    worker_threads.times do
-      Thread.new do
-        Thread.current[:yahns_rbuf] = ""
-        begin
-          epoll_wait(max_events) do |_, io| # don't care for flags for now
-            case rv = io.yahns_step
-            when :wait_readable
-              epoll_ctl(Epoll::CTL_MOD, io, QEV_RD)
-            when :wait_writable
-              epoll_ctl(Epoll::CTL_MOD, io, QEV_WR)
-            when :wait_readwrite
-              epoll_ctl(Epoll::CTL_MOD, io, QEV_RDWR)
-            when :delete # only used by rack.hijack
-              epoll_ctl(Epoll::CTL_DEL, io, 0)
-              @fdmap.delete(io)
-            when nil
-              # this is be the ONLY place where we call IO#close on
-              # things inside the queue
-              io.close
-              @fdmap.decr
-            else
-              raise "BUG: #{io.inspect}#yahns_step returned: #{rv.inspect}"
-            end
+  def worker_thread(logger, max_events)
+    Thread.new do
+      Thread.current[:yahns_rbuf] = ""
+      begin
+        epoll_wait(max_events) do |_, io| # don't care for flags for now
+          case rv = io.yahns_step
+          when :wait_readable
+            epoll_ctl(Epoll::CTL_MOD, io, QEV_RD)
+          when :wait_writable
+            epoll_ctl(Epoll::CTL_MOD, io, QEV_WR)
+          when :wait_readwrite
+            epoll_ctl(Epoll::CTL_MOD, io, QEV_RDWR)
+          when :ignore # only used by rack.hijack
+            @fdmap.decr
+          when nil
+            # this is be the ONLY place where we call IO#close on
+            # things inside the queue
+            io.close
+            @fdmap.decr
+          else
+            raise "BUG: #{io.inspect}#yahns_step returned: #{rv.inspect}"
           end
-        rescue => e
-          break if (IOError === e || Errno::EBADF === e) && closed?
-          Yahns::Log.exception(logger, 'queue loop', e)
-        end while true
-      end
+        end
+      rescue => e
+        # sleep since this check is racy (and uncommon)
+        break if closed? || (sleep(0.01) && closed?)
+        Yahns::Log.exception(logger, 'queue loop', e)
+      end while true
     end
   end
 end