yahns 0.0.0TP1

data/lib/yahns/tee_input.rb

@@ -0,0 +1,114 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+
+# acts like tee(1) on an input input to provide a input-like stream
+# while providing rewindable semantics through a File/StringIO backing
+# store.  On the first pass, the input is only read on demand so your
+# Rack application can use input notification (upload progress and
+# like).  This should fully conform to the Rack::Lint::InputWrapper
+# specification on the public API.  This class is intended to be a
+# strict interpretation of Rack::Lint::InputWrapper functionality and
+# will not support any deviations from it.
+#
+# When processing uploads, Yahns exposes a TeeInput object under
+# "rack.input" of the Rack environment.
+class Yahns::TeeInput < Yahns::StreamInput # :nodoc:
+  # Initializes a new TeeInput object.  You normally do not have to call
+  # this unless you are writing an HTTP server.
+  def initialize(client, request)
+    @len = request.content_length
+    super
+    @tmp = client.class.tmpio_for(@len)
+  end
+
+  # :call-seq:
+  #   ios.size  => Integer
+  #
+  # Returns the size of the input.  For requests with a Content-Length
+  # header value, this will not read data off the socket and just return
+  # the value of the Content-Length header as an Integer.
+  #
+  # For Transfer-Encoding:chunked requests, this requires consuming
+  # all of the input stream before returning since there's no other
+  # way to determine the size of the request body beforehand.
+  #
+  # This method is no longer part of the Rack specification as of
+  # Rack 1.2, so its use is not recommended.  This method only exists
+  # for compatibility with Rack applications designed for Rack 1.1 and
+  # earlier.  Most applications should only need to call +read+ with a
+  # specified +length+ in a loop until it returns +nil+.
+  def size
+    @len and return @len
+    pos = @tmp.pos
+    consume!
+    @tmp.pos = pos
+    @len = @tmp.size
+  end
+
+  # :call-seq:
+  #   ios.read([length [, buffer ]]) => string, buffer, or nil
+  #
+  # Reads at most length bytes from the I/O stream, or to the end of
+  # file if length is omitted or is nil. length must be a non-negative
+  # integer or nil. If the optional buffer argument is present, it
+  # must reference a String, which will receive the data.
+  #
+  # At end of file, it returns nil or "" depend on length.
+  # ios.read() and ios.read(nil) returns "".
+  # ios.read(length [, buffer]) returns nil.
+  #
+  # If the Content-Length of the HTTP request is known (as is the common
+  # case for POST requests), then ios.read(length [, buffer]) will block
+  # until the specified length is read (or it is the last chunk).
+  # Otherwise, for uncommon "Transfer-Encoding: chunked" requests,
+  # ios.read(length [, buffer]) will return immediately if there is
+  # any data and only block when nothing is available (providing
+  # IO#readpartial semantics).
+  def read(*args)
+    @client ? tee(super) : @tmp.read(*args)
+  end
+
+  # :call-seq:
+  #   ios.gets   => string or nil
+  #
+  # Reads the next ``line'' from the I/O stream; lines are separated
+  # by the global record separator ($/, typically "\n"). A global
+  # record separator of nil reads the entire unread contents of ios.
+  # Returns nil if called at the end of file.
+  # This takes zero arguments for strict Rack::Lint compatibility,
+  # unlike IO#gets.
+  def gets
+    @client ? tee(super) : @tmp.gets
+  end
+
+  # :call-seq:
+  #   ios.rewind    => 0
+  #
+  # Positions the *ios* pointer to the beginning of input, returns
+  # the offset (zero) of the +ios+ pointer.  Subsequent reads will
+  # start from the beginning of the previously-buffered input.
+  def rewind
+    return 0 if 0 == @tmp.size
+    consume! if @client
+    @tmp.rewind # Rack does not specify what the return value is here
+  end
+
+  # consumes the stream of the socket
+  def consume!
+    junk = ""
+    rsize = __rsize
+    nil while read(rsize, junk)
+  end
+
+  def tee(buffer)
+    if buffer && buffer.size > 0
+      @tmp.write(buffer)
+    end
+    buffer
+  end
+
+  def close # returns nil
+    @tmp = @tmp.close
+  end
+end

data/lib/yahns/tmpio.rb

@@ -0,0 +1,27 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+require 'tmpdir'
+
+# some versions of Ruby had a broken Tempfile which didn't work
+# well with unlinked files.  This one is much shorter, easier
+# to understand, and slightly faster (no delegation).
+class Yahns::TmpIO < File # :nodoc:
+
+  # creates and returns a new File object.  The File is unlinked
+  # immediately, switched to binary mode, and userspace output
+  # buffering is disabled
+  def self.new
+    fp = begin
+      super("#{Dir.tmpdir}/#{rand}", RDWR|CREAT|EXCL, 0600)
+    rescue Errno::EEXIST
+      retry
+    end
+    unlink(fp.path)
+    fp.binmode
+    fp.sync = true
+    fp
+  end
+
+  alias discard close
+end

data/lib/yahns/wbuf.rb

@@ -0,0 +1,36 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'wbuf_common'
+
+class Yahns::Wbuf # :nodoc:
+  include Yahns::WbufCommon
+
+  def initialize(body, persist)
+    @tmpio = Yahns::TmpIO.new
+    @sf_offset = @sf_count = 0
+    @wbuf_persist = persist # whether or not we keep the connection alive
+    @body = body
+  end
+
+  def wbuf_write(client, buf)
+    @sf_count += @tmpio.write(buf)
+    case rv = client.trysendfile(@tmpio, @sf_offset, @sf_count)
+    when Integer
+      @sf_count -= rv
+      @sf_offset += rv
+    when :wait_writable, :wait_readable
+      return rv
+    else
+      raise "BUG: #{rv.nil ? "EOF" : rv.inspect} on tmpio " \
+            "sf_offset=#@sf_offset sf_count=#@sf_count"
+    end while @sf_count > 0
+    nil
+  end
+
+  # called by last wbuf_flush
+  def wbuf_close(client)
+    @tmpio = @tmpio.close
+    wbuf_close_common(client)
+  end
+end

data/lib/yahns/wbuf_common.rb

@@ -0,0 +1,32 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require 'sendfile'
+module Yahns::WbufCommon # :nodoc:
+  # returns nil on success, :wait_*able when blocked
+  # currently, we rely on each thread having exclusive access to the
+  # client socket, so this is never called concurrently with wbuf_write
+  def wbuf_flush(client)
+    case rv = client.trysendfile(@tmpio, @sf_offset, @sf_count)
+    when Integer
+      return wbuf_close(client) if (@sf_count -= rv) == 0 # all sent!
+
+      @sf_offset += rv # keep going otherwise
+    when :wait_writable, :wait_readable
+      return rv
+    else
+      raise "BUG: #{rv.nil? ? "EOF" : rv.inspect} on tmpio=#{@tmpio.inspect} " \
+            "sf_offset=#@sf_offset sf_count=#@sf_count"
+    end while true
+  end
+
+  def wbuf_close_common(client)
+    @body.close if @body.respond_to?(:close)
+    if @wbuf_persist.respond_to?(:call) # hijack
+      @wbuf_persist.call(client)
+      :delete
+    else
+      @wbuf_persist # true or false or Yahns::StreamFile
+    end
+  end
+end

data/lib/yahns/worker.rb

@@ -0,0 +1,58 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+class Yahns::Worker # :nodoc:
+  attr_accessor :nr
+  attr_reader :to_io
+
+  def initialize(nr)
+    @nr = nr
+    @to_io, @wr = Kgio::Pipe.new
+  end
+
+  def atfork_child
+    @wr = @wr.close # nil @wr to save space in worker process
+  end
+
+  def atfork_parent
+    @to_io = @to_io.close
+    self
+  end
+
+  # used in the worker process.
+  # This causes the worker to gracefully exit if the master
+  # dies unexpectedly.
+  def yahns_step
+    @to_io.kgio_tryread(11) == nil and Process.kill(:QUIT, $$)
+    :wait_readable
+  end
+
+  # worker objects may be compared to just plain Integers
+  def ==(other_nr) # :nodoc:
+    @nr == other_nr
+  end
+
+  # Changes the worker process to the specified +user+ and +group+
+  # This is only intended to be called from within the worker
+  # process from the +after_fork+ hook.  This should be called in
+  # the +after_fork+ hook after any privileged functions need to be
+  # run (e.g. to set per-worker CPU affinity, niceness, etc)
+  #
+  # Any and all errors raised within this method will be propagated
+  # directly back to the caller (usually the +after_fork+ hook.
+  # These errors commonly include ArgumentError for specifying an
+  # invalid user/group and Errno::EPERM for insufficient privileges
+  def user(user, group = nil)
+    # we do not protect the caller, checking Process.euid == 0 is
+    # insufficient because modern systems have fine-grained
+    # capabilities.  Let the caller handle any and all errors.
+    uid = Etc.getpwnam(user).uid
+    gid = Etc.getgrnam(group).gid if group
+    Yahns::Log.chown_all(uid, gid)
+    if gid && Process.egid != gid
+      Process.initgroups(user, gid)
+      Process::GID.change_privilege(gid)
+    end
+    Process.euid != uid and Process::UID.change_privilege(uid)
+  end
+end

data/test/covshow.rb

@@ -0,0 +1,29 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+# this works with the __covmerge method in test/helper.rb
+# run this file after all tests are run
+
+# load the merged dump data
+res = Marshal.load(IO.binread("coverage.dump"))
+
+# Dirty little text formatter.  I tried simplecov but the default
+# HTML+JS is unusable without a GUI (I hate GUIs :P) and it would've
+# taken me longer to search the Internets to find a plain-text
+# formatter I like...
+res.keys.sort.each do |filename|
+  cov = res[filename]
+  puts "==> #{filename} <=="
+  File.readlines(filename).each_with_index do |line, i|
+    n = cov[i]
+    if n == 0 # BAD
+      print("  *** 0 #{line}")
+    elsif n
+      printf("% 7u %s", n, line)
+    elsif line =~ /\S/ # probably a line with just "end" in it
+      print("        #{line}")
+    else # blank line
+      print "\n" # don't output trailing whitespace on blank lines
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,115 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+$stdout.sync = $stderr.sync = Thread.abort_on_exception = true
+require 'thread'
+
+# Global Test Lock, to protect:
+#   Process.wait*, Dir.chdir, ENV, trap, require, etc...
+GTL = Mutex.new
+
+# fork-aware coverage data gatherer, see also test/covshow.rb
+if ENV["COVERAGE"]
+  require "coverage"
+  COVMATCH = %r{/lib/yahns\b.*rb\z}
+  COVTMP = File.open("coverage.dump", IO::CREAT|IO::RDWR)
+  COVTMP.binmode
+  COVTMP.sync = true
+
+  def __covmerge
+    res = Coverage.result
+
+    # we own this file (at least until somebody tries to use NFS :x)
+    COVTMP.flock(File::LOCK_EX)
+
+    COVTMP.rewind
+    prev = COVTMP.read
+    prev = prev.empty? ? {} : Marshal.load(prev)
+    res.each do |filename, counts|
+      # filter out stuff that's not in our project
+      COVMATCH =~ filename or next
+
+      merge = prev[filename] || []
+      merge = merge
+      counts.each_with_index do |count, i|
+        count or next
+        merge[i] = (merge[i] || 0) + count
+      end
+      prev[filename] = merge
+    end
+    COVTMP.rewind
+    COVTMP.truncate(0)
+    COVTMP.write(Marshal.dump(prev))
+    COVTMP.flock(File::LOCK_UN)
+  end
+
+  Coverage.start
+  at_exit { at_exit { __covmerge } }
+end
+
+gem 'minitest'
+require 'minitest/autorun'
+require "tempfile"
+
+Testcase = begin
+  Minitest::Test # minitest 5
+rescue NameError
+  Minitest::Unit::TestCase # minitest 4
+end
+
+FIFOS = []
+def tmpfifo
+  tmp = Tempfile.new(%w(yahns-test .fifo))
+  path = tmp.path
+  tmp.close!
+  assert system(*%W(mkfifo #{path})), "mkfifo #{path}"
+
+  GTL.synchronize do
+    if FIFOS.empty?
+      at_exit do
+        FIFOS.each { |(pid,_path)| File.unlink(_path) if $$ == pid }
+      end
+    end
+    FIFOS << [ $$, path ]
+  end
+  path
+end
+
+require 'tmpdir'
+class Dir
+  require 'fileutils'
+  def Dir.mktmpdir
+    begin
+      d = "#{Dir.tmpdir}/#$$.#{rand}"
+      Dir.mkdir(d)
+    rescue Errno::EEXIST
+    end while true
+    begin
+      yield d
+    ensure
+      FileUtils.remove_entry(d)
+    end
+  end
+end unless Dir.respond_to?(:mktmpdir)
+
+def tmpfile(*args)
+  tmp = Tempfile.new(*args)
+  tmp.sync = true
+  tmp.binmode
+  tmp
+end
+
+require 'io/wait'
+# needed for Rubinius 2.0.0, we only use IO#nread in tests
+class IO
+  # this ignores buffers
+  def nread
+    buf = "\0" * 8
+    ioctl(0x541B, buf)
+    buf.unpack("l_")[0]
+  end
+end if ! IO.method_defined?(:nread) && RUBY_PLATFORM =~ /linux/
+
+require 'yahns'
+
+# needed for parallel (MT) tests)
+require 'yahns/rack'

data/test/server_helper.rb

@@ -0,0 +1,65 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'helper'
+require 'timeout'
+require 'socket'
+require 'net/http'
+
+module ServerHelper
+  def check_err(err = @err)
+    err = File.open(err.path, "r") if err.respond_to?(:path)
+    err.rewind
+    lines = err.readlines.delete_if { |l| l =~ /INFO/ }
+    assert lines.empty?, lines.join("\n")
+    err.close! if err == @err
+  end
+
+  def poke_until_dead(pid)
+    Timeout.timeout(10) do
+      begin
+        Process.kill(0, pid)
+        sleep(0.01)
+      rescue Errno::ESRCH
+        break
+      end while true
+    end
+    assert_raises(Errno::ESRCH) { Process.kill(0, pid) }
+  end
+
+  def quit_wait(pid)
+    pid or return
+    Process.kill(:QUIT, pid)
+    _, status = Timeout.timeout(10) { Process.waitpid2(pid) }
+    assert status.success?, status.inspect
+  rescue Timeout::Error
+    if RUBY_PLATFORM =~ /linux/
+      system("lsof -p #{pid}")
+      warn "#{pid} failed to die, waiting for user to inspect"
+      sleep
+    end
+    raise
+  end
+
+  # only use for newly bound sockets
+  def get_tcp_client(host, port, tries = 500)
+    begin
+      c = TCPSocket.new(host, port)
+      return c
+    rescue Errno::ECONNREFUSED
+      raise if tries < 0
+      tries -= 1
+    end while sleep(0.01)
+  end
+
+  def server_helper_teardown
+    @srv.close unless @srv.closed?
+    @ru.close! if @ru
+    check_err
+  end
+
+  def server_helper_setup
+    @srv = TCPServer.new(ENV["TEST_HOST"] || "127.0.0.1", 0)
+    @err = tmpfile(%w(srv .err))
+    @ru = nil
+  end
+end