yahns 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a9749686719fc4ebab52f18c57c3c3b9cf00ab8
-  data.tar.gz: 82df4e6686e4ee97cad0462fb1d0c809d19f9a10
+  metadata.gz: 7b73379a257ad0e09cd00b6d1b1b316ca6913992
+  data.tar.gz: 1e5203cb1d70720bdab3f9624b9766e3d49a9d51
 SHA512:
-  metadata.gz: f5f385fabee64576cadcea538d77065ec625e50ee6c9c40ebba09d662995f6ad9c1740a422ee219078c1f1a7c7198120927625d08632b5d16d6f96dfd8039fb3
-  data.tar.gz: 12080fba5d268e326f648da62708ec2927063b2de470a3590be6d55d21968ea5150e94d8b18165dba22ca9388fbb535c9f5180039926a75df372d4f602d60c76
+  metadata.gz: 1c7a49cdbe694c1a27bd3f26d6db10b4b1f8ac11cf9307036957ba9df60aab266ef621cf515dcfd9eb592675648a0864b9949ba9b11ffccf083627a392353879
+  data.tar.gz: 2250ca7bf3fdfffbbb42cbb332cbe08bca48aa4c0e6066d3414e3248a2e666094ba2da07ff27312021558b67e45de59bdb10428fcd089c72bbfaeabd9f3488da

data/GIT-VERSION-GEN

@@ -4,7 +4,7 @@
 CONSTANT = "Yahns::VERSION"
 RVF = "lib/yahns/version.rb"
 GVF = "GIT-VERSION-FILE"
-DEF_VER = "v1.1.0"
+DEF_VER = "v1.2.0"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/INSTALL

@@ -6,3 +6,10 @@ No tarballs are currently provided.
 
 You may also install yahns from the git source, see the HACKING document for
 more details.
+
+Debian GNU/kFreeBSD:
+
+For now, you will need to define SENDFILE_BROKEN=1 in the env before
+running yahns, and also install the "io-extra" RubyGem (tested on 1.2.7)
+
+	gem install -v 1.2.7 io-extra

data/README

@@ -44,11 +44,11 @@ Supported Platforms
 
 yahns is developed primarily for modern GNU/Linux systems.
 
-We may support kqueue for FreeBSD/OpenBSD/NetBSD if there is significant
-interest.  Non-Free systems/dependencies will never be supported
+We have experimental support kqueue on FreeBSD (and possibly OpenBSD and
+NetBSD).  Non-Free systems/dependencies will never be supported.
 
 Supported Ruby implementations:
-* (Matz) Ruby 1.9.3 and later (we develop against trunk)
+* (Matz) Ruby 1.9.3 and later (we develop (and host our website) on trunk)
 * Rubinius 2.0 or later (best-effort)
 
 Contact

data/lib/yahns.rb

@@ -30,7 +30,7 @@ module Yahns # :nodoc:
   #
   #   Yahns::START[0] = "/home/bofh/2.0.0/bin/yahns"
   START = {
-    :argv => ARGV.map { |arg| arg.dup },
+    :argv => ARGV.map(&:dup),
     0 => $0.dup,
   }
 
@@ -57,7 +57,7 @@ end
 
 # FIXME: require lazily
 require_relative 'yahns/log'
-require_relative 'yahns/queue_epoll'
+require_relative 'yahns/queue'
 require_relative 'yahns/stream_input'
 require_relative 'yahns/tee_input'
 require_relative 'yahns/queue_egg'

data/lib/yahns/queue_kqueue.rb

@@ -0,0 +1,82 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2014, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+# This is the dangerous, low-level kqueue interface for sleepy_penguin
+# It is safe as long as you're aware of all potential concurrency
+# issues given multithreading, GC, and kqueue itself.
+class Yahns::Queue < SleepyPenguin::Kqueue::IO # :nodoc:
+  include SleepyPenguin
+  attr_accessor :fdmap # Yahns::Fdmap
+
+  # public
+  QEV_QUIT = nil # Level Trigger for QueueQuitter
+  QEV_RD = EvFilt::READ
+  QEV_WR = EvFilt::WRITE
+
+  ADD_ONESHOT = Ev::ADD | Ev::ONESHOT # private
+
+  def self.new
+    rv = super
+    rv.close_on_exec = true
+    rv
+  end
+
+  # for HTTP and HTTPS servers, we rely on the io writing to us, first
+  # flags: QEV_RD/QEV_WR (usually QEV_RD)
+  def queue_add(io, flags)
+    # order is very important here, this thread cannot do anything with
+    # io once we've issued epoll_ctl() because another thread may use it
+    @fdmap.add(io)
+    fflags = ADD_ONESHOT
+    if flags == QEV_QUIT
+      fflags = Ev::ADD
+      flags = QEV_WR
+    end
+    kevent(Kevent[io.fileno, flags, fflags, 0, 0, io])
+  end
+
+  def thr_init
+    Thread.current[:yahns_rbuf] = ""
+    Thread.current[:yahns_fdmap] = @fdmap
+  end
+
+  def queue_del(io)
+    # do not bother with kevent EV_DELETE, it may be tricky to get right,
+    # we only did it in epoll since Eric knows the epoll internals well.
+    @fdmap.forget(io)
+  end
+
+  # returns an array of infinitely running threads
+  def worker_thread(logger, max_events)
+    Thread.new do
+      thr_init
+      begin
+        kevent(nil, max_events) do |_,_,_,_,_,io| # don't care for flags for now
+          # Note: we absolutely must not do anything with io after
+          # we've called epoll_ctl on it, io is exclusive to this
+          # thread only until epoll_ctl is called on it.
+          case rv = io.yahns_step
+          when :wait_readable
+            kevent(Kevent[io.fileno, QEV_RD, ADD_ONESHOT, 0, 0, io])
+          when :wait_writable
+            kevent(Kevent[io.fileno, QEV_WR, ADD_ONESHOT, 0, 0, io])
+          when :ignore # only used by rack.hijack
+            # we cannot EV_DELETE after hijacking, the hijacker
+            # may have already closed it  Likewise, io.fileno is not
+            # expected to work, so we had to erase it from fdmap before hijack
+          when nil, :close
+            # this must be the ONLY place where we call IO#close on
+            # things that got inside the queue
+            @fdmap.sync_close(io)
+          else
+            raise "BUG: #{io.inspect}#yahns_step returned: #{rv.inspect}"
+          end
+        end
+      rescue => e
+        break if closed? # can still happen due to shutdown_timeout
+        Yahns::Log.exception(logger, 'queue loop', e)
+      end while true
+    end
+  end
+end

data/lib/yahns/queue_quitter_pipe.rb

@@ -5,9 +5,8 @@
 class Yahns::QueueQuitter # :nodoc:
   attr_reader :to_io
   def initialize
-    reader, @to_io = IO.pipe
+    @reader, @to_io = IO.pipe
     @to_io.close_on_exec = true
-    reader.close
   end
 
   def yahns_step
@@ -19,6 +18,7 @@ class Yahns::QueueQuitter # :nodoc:
   end
 
   def close
+    @reader.close
     @to_io.close
   end
 end

data/lib/yahns/sendfile_compat.rb

@@ -0,0 +1,29 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2014, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require 'io/extra' # gem install io-extra
+
+module Yahns::SendfileCompat
+  def trysendfile(io, offset, count)
+    return 0 if count == 0
+    count = 0x4000 if count > 0x4000
+    str = IO.pread(io.fileno, count, offset)
+    if count > str.bytesize
+      raise EOFError, "end of file reached"
+    end
+    n = 0
+    case rv = kgio_trywrite(str)
+    when String # partial write, keep trying
+      n += (str.bytesize - rv.bytesize)
+      str = rv
+    when :wait_writable, :wait_readable
+      return n > 0 ? n : rv
+    when nil
+      return n + str.bytesize # yay!
+    end while true
+  end
+end
+
+class IO
+  include Yahns::SendfileCompat
+end

data/lib/yahns/sigevent_pipe.rb

@@ -8,8 +8,8 @@ class Yahns::Sigevent # :nodoc:
     @to_io.close_on_exec = @wr.close_on_exec = true
   end
 
-  def kgio_wait_readable
-    @to_io.kgio_wait_readable
+  def kgio_wait_readable(*args)
+    @to_io.kgio_wait_readable(*args)
   end
 
   def sev_signal

data/lib/yahns/wbuf.rb

@@ -30,6 +30,14 @@ require_relative 'wbuf_common'
 class Yahns::Wbuf # :nodoc:
   include Yahns::WbufCommon
 
+  # TODO: Figure out why this hack is needed to pass output buffering tests.
+  # It could be a bug in our code, Ruby, the sendfile gem, or FreeBSD itself.
+  # Tested on FreeBSD fbsd 9.2-RELEASE FreeBSD 9.2-RELEASE #0 r255898
+  # We are able to use bypass mode on Linux to reduce buffering in some
+  # cases.  Without bypass mode, we must always finish writing the entire
+  # response completely before sending more data to the client.
+  bypass_ok = RUBY_PLATFORM =~ /linux/
+
   def initialize(body, persist, tmpdir)
     @tmpio = Yahns::TmpIO.new(tmpdir)
     @sf_offset = @sf_count = 0
@@ -67,7 +75,12 @@ class Yahns::Wbuf # :nodoc:
     @tmpio.rewind
     @bypass = true
     nil
-  end
+  end if bypass_ok
+
+  def wbuf_write(client, buf)
+    @sf_count += @tmpio.write(buf)
+    :wait_writable
+  end unless bypass_ok
 
   # called by last wbuf_flush
   def wbuf_close(client)

data/lib/yahns/wbuf_common.rb

@@ -1,7 +1,12 @@
 # -*- encoding: binary -*-
 # Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
 # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
-require 'sendfile'
+if ENV["SENDFILE_BROKEN"]
+  require_relative 'sendfile_compat'
+else
+  require 'sendfile'
+end
+
 module Yahns::WbufCommon # :nodoc:
   # returns nil on success, :wait_*able when blocked
   # currently, we rely on each thread having exclusive access to the
@@ -25,7 +30,8 @@ module Yahns::WbufCommon # :nodoc:
       raise "BUG: rv=#{rv.inspect} " \
             "on tmpio=#{@tmpio.inspect} " \
             "sf_offset=#@sf_offset sf_count=#@sf_count"
-    end while true
+    end while @sf_count > 0
+    wbuf_close(client)
   end
 
   def wbuf_close_common(client)

data/test/helper.rb

@@ -36,6 +36,12 @@ if ENV["COVERAGE"]
         # filter out stuff that's not in our project
         COVMATCH =~ filename or next
 
+        # For compatibility with https://bugs.ruby-lang.org/issues/9508
+        # TODO: support those features if that gets merged into mainline
+        unless Array === counts
+          counts = counts[:lines]
+        end
+
         merge = prev[filename] || []
         merge = merge
         counts.each_with_index do |count, i|

data/test/test_buffer_tmpdir.rb

@@ -9,12 +9,18 @@ class TestBufferTmpdir < Testcase
   attr_reader :ino, :tmpdir
 
   def setup
-    @ino = SleepyPenguin::Inotify.new(:CLOEXEC)
+    @ino = nil
+    begin
+      @ino = SleepyPenguin::Inotify.new(:CLOEXEC)
+    rescue
+      skip "test needs inotify"
+    end
     @tmpdir = Dir.mktmpdir
     server_helper_setup
   end
 
   def teardown
+    return unless @ino
     server_helper_teardown
     @ino.close
     FileUtils.rm_rf @tmpdir
@@ -100,4 +106,4 @@ class TestBufferTmpdir < Testcase
     c.close if c
     quit_wait(pid)
   end
-end if SleepyPenguin.const_defined?(:Inotify)
+end

data/test/test_client_expire.rb

@@ -149,9 +149,17 @@ class TestClientExpire < Testcase
         Process.waitpid2(_pid)
       end
     end
+
+    # this seems to be needed in Debian GNU/kFreeBSD
+    linux = !!(RUBY_PLATFORM =~ /linux/)
+    sleep(1) unless linux
+
     [ f, s ].each do |io|
       assert_raises(Errno::EPIPE,Errno::ECONNRESET) do
-        req.each_byte { |b| io.write(b.chr) }
+        req.each_byte do |b|
+          io.write(b.chr)
+          sleep(0.01) unless linux
+        end
       end
       io.close
     end

data/test/test_mt_accept.rb

@@ -10,6 +10,8 @@ class TestMtAccept < Testcase
   alias teardown server_helper_teardown
 
   def test_mt_accept
+    skip "Linux kernel required" unless RUBY_PLATFORM =~ /linux/
+    skip "/proc not mounted" unless File.directory?("/proc")
     err, cfg, host, port = @err, Yahns::Config.new, @srv.addr[3], @srv.addr[1]
     opts = { threads: 1 }
     cfg.instance_eval do
@@ -45,4 +47,4 @@ class TestMtAccept < Testcase
   ensure
     quit_wait(pid)
   end
-end if RUBY_PLATFORM =~ /linux/ && File.directory?("/proc")
+end

data/test/test_server.rb

@@ -178,11 +178,14 @@ class TestServer < Testcase
   end
 
   def test_check_client_connection
+    tmpdir = Dir.mktmpdir
+    sock = "#{tmpdir}/sock"
+    unix_srv = UNIXServer.new(sock)
+    unix_srv.close_on_exec = true
     msgs = %w(ZZ zz)
     err = @err
     cfg = Yahns::Config.new
     bpipe = cloexec_pipe
-    host, port = @srv.addr[3], @srv.addr[1]
     cfg.instance_eval do
       ru = lambda { |e|
         case e['PATH_INFO']
@@ -205,7 +208,7 @@ class TestServer < Testcase
       }
       GTL.synchronize {
         app(:rack, ru) {
-          listen "#{host}:#{port}"
+          listen sock
           check_client_connection true
           # needed to avoid concurrency with check_client_connection
           queue { worker_threads 1 }
@@ -223,12 +226,13 @@ class TestServer < Testcase
 
     pid = fork do
       bpipe[1].close
-      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      ENV["YAHNS_FD"] = unix_srv.fileno.to_s
       srv.start.join
     end
     bpipe[0].close
-    a = get_tcp_client(host, port)
-    b = get_tcp_client(host, port)
+    a = UNIXSocket.new(sock)
+    b = UNIXSocket.new(sock)
+    b.close_on_exec = a.close_on_exec = true
     a.write("GET /sleep HTTP/1.0\r\n\r\n")
     r = IO.select([a], nil, nil, 4)
     assert r, "nothing ready"
@@ -250,10 +254,19 @@ class TestServer < Testcase
     assert_equal msgs.join, buf.split(/\r\n\r\n/)[1]
 
     # do things still work?
-    run_client(host, port) { |res| assert_equal "HI", res.body }
+    c = UNIXSocket.new(sock)
+    c.write "GET /\r\n\r\n"
+    assert_equal "HI", c.read
+    c.close
     a.close
+  rescue => e
+    warn e.class
+    warn e.message
+    warn e.backtrace.join("\n")
   ensure
+    unix_srv.close
     quit_wait(pid)
+    FileUtils.rm_rf(tmpdir)
   end
 
   def test_mp

data/test/test_wbuf.rb

@@ -81,8 +81,8 @@ class TestWbuf < Testcase
     assert_equal b, IO.select([b], nil, nil, 5)[0][0]
     b.read(nr - 2) if nr > 2
     assert_equal b, IO.select([b], nil, nil, 5)[0][0]
-    assert_equal "HI", b.read(2)
-    assert_equal false, wbuf.wbuf_flush(a)
+    assert_equal "HI", b.read(2), "read the end of the response"
+    assert_equal true, wbuf.wbuf_flush(a)
   ensure
     a.close
     b.close

data/yahns.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.files = manifest
   s.add_dependency(%q<kgio>, '~> 2.9')
   s.add_dependency(%q<sleepy_penguin>, '~> 3.2')
-  s.add_dependency(%q<sendfile>, '~> 1.2.1')
+  s.add_dependency(%q<kgio-sendfile>, '~> 1.2')
   s.add_dependency(%q<unicorn>, '~> 4.6', '>= 4.6.3')
 
   # minitest is standard in Ruby 2.0, 4.3 is packaged with Ruby 2.0.0,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yahns
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - yahns hackers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-04 00:00:00.000000000 Z
+date: 2014-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kgio
@@ -39,19 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.2'
 - !ruby/object:Gem::Dependency
-  name: sendfile
+  name: kgio-sendfile
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: unicorn
   requirement: !ruby/object:Gem::Requirement
@@ -152,10 +152,12 @@ files:
 - lib/yahns/queue.rb
 - lib/yahns/queue_egg.rb
 - lib/yahns/queue_epoll.rb
+- lib/yahns/queue_kqueue.rb
 - lib/yahns/queue_quitter.rb
 - lib/yahns/queue_quitter_pipe.rb
 - lib/yahns/rack.rb
 - lib/yahns/rackup_handler.rb
+- lib/yahns/sendfile_compat.rb
 - lib/yahns/server.rb
 - lib/yahns/server_mp.rb
 - lib/yahns/sigevent.rb
@@ -223,7 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: sleepy, multi-threaded, non-blocking application server