yahns 0.0.0TP1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/COPYING +674 -0
  4. data/GIT-VERSION-GEN +41 -0
  5. data/GNUmakefile +90 -0
  6. data/README +127 -0
  7. data/Rakefile +60 -0
  8. data/bin/yahns +32 -0
  9. data/examples/README +3 -0
  10. data/examples/init.sh +76 -0
  11. data/examples/logger_mp_safe.rb +28 -0
  12. data/examples/logrotate.conf +32 -0
  13. data/examples/yahns_multi.conf.rb +89 -0
  14. data/examples/yahns_rack_basic.conf.rb +27 -0
  15. data/lib/yahns.rb +73 -0
  16. data/lib/yahns/acceptor.rb +28 -0
  17. data/lib/yahns/client_expire.rb +40 -0
  18. data/lib/yahns/client_expire_portable.rb +39 -0
  19. data/lib/yahns/config.rb +344 -0
  20. data/lib/yahns/daemon.rb +51 -0
  21. data/lib/yahns/fdmap.rb +90 -0
  22. data/lib/yahns/http_client.rb +198 -0
  23. data/lib/yahns/http_context.rb +65 -0
  24. data/lib/yahns/http_response.rb +184 -0
  25. data/lib/yahns/log.rb +73 -0
  26. data/lib/yahns/queue.rb +7 -0
  27. data/lib/yahns/queue_egg.rb +23 -0
  28. data/lib/yahns/queue_epoll.rb +57 -0
  29. data/lib/yahns/rack.rb +80 -0
  30. data/lib/yahns/server.rb +336 -0
  31. data/lib/yahns/server_mp.rb +181 -0
  32. data/lib/yahns/sigevent.rb +7 -0
  33. data/lib/yahns/sigevent_efd.rb +18 -0
  34. data/lib/yahns/sigevent_pipe.rb +29 -0
  35. data/lib/yahns/socket_helper.rb +117 -0
  36. data/lib/yahns/stream_file.rb +34 -0
  37. data/lib/yahns/stream_input.rb +150 -0
  38. data/lib/yahns/tee_input.rb +114 -0
  39. data/lib/yahns/tmpio.rb +27 -0
  40. data/lib/yahns/wbuf.rb +36 -0
  41. data/lib/yahns/wbuf_common.rb +32 -0
  42. data/lib/yahns/worker.rb +58 -0
  43. data/test/covshow.rb +29 -0
  44. data/test/helper.rb +115 -0
  45. data/test/server_helper.rb +65 -0
  46. data/test/test_bin.rb +97 -0
  47. data/test/test_client_expire.rb +132 -0
  48. data/test/test_config.rb +56 -0
  49. data/test/test_fdmap.rb +19 -0
  50. data/test/test_output_buffering.rb +291 -0
  51. data/test/test_queue.rb +59 -0
  52. data/test/test_rack.rb +28 -0
  53. data/test/test_serve_static.rb +42 -0
  54. data/test/test_server.rb +415 -0
  55. data/test/test_stream_file.rb +30 -0
  56. data/test/test_wbuf.rb +136 -0
  57. data/yahns.gemspec +19 -0
  58. metadata +165 -0
@@ -0,0 +1,114 @@
1
+ # -*- encoding: binary -*-
2
+ # Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
3
+ # License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
4
+
5
+ # acts like tee(1) on an input input to provide a input-like stream
6
+ # while providing rewindable semantics through a File/StringIO backing
7
+ # store. On the first pass, the input is only read on demand so your
8
+ # Rack application can use input notification (upload progress and
9
+ # like). This should fully conform to the Rack::Lint::InputWrapper
10
+ # specification on the public API. This class is intended to be a
11
+ # strict interpretation of Rack::Lint::InputWrapper functionality and
12
+ # will not support any deviations from it.
13
+ #
14
+ # When processing uploads, Yahns exposes a TeeInput object under
15
+ # "rack.input" of the Rack environment.
16
+ class Yahns::TeeInput < Yahns::StreamInput # :nodoc:
17
+ # Initializes a new TeeInput object. You normally do not have to call
18
+ # this unless you are writing an HTTP server.
19
+ def initialize(client, request)
20
+ @len = request.content_length
21
+ super
22
+ @tmp = client.class.tmpio_for(@len)
23
+ end
24
+
25
+ # :call-seq:
26
+ # ios.size => Integer
27
+ #
28
+ # Returns the size of the input. For requests with a Content-Length
29
+ # header value, this will not read data off the socket and just return
30
+ # the value of the Content-Length header as an Integer.
31
+ #
32
+ # For Transfer-Encoding:chunked requests, this requires consuming
33
+ # all of the input stream before returning since there's no other
34
+ # way to determine the size of the request body beforehand.
35
+ #
36
+ # This method is no longer part of the Rack specification as of
37
+ # Rack 1.2, so its use is not recommended. This method only exists
38
+ # for compatibility with Rack applications designed for Rack 1.1 and
39
+ # earlier. Most applications should only need to call +read+ with a
40
+ # specified +length+ in a loop until it returns +nil+.
41
+ def size
42
+ @len and return @len
43
+ pos = @tmp.pos
44
+ consume!
45
+ @tmp.pos = pos
46
+ @len = @tmp.size
47
+ end
48
+
49
+ # :call-seq:
50
+ # ios.read([length [, buffer ]]) => string, buffer, or nil
51
+ #
52
+ # Reads at most length bytes from the I/O stream, or to the end of
53
+ # file if length is omitted or is nil. length must be a non-negative
54
+ # integer or nil. If the optional buffer argument is present, it
55
+ # must reference a String, which will receive the data.
56
+ #
57
+ # At end of file, it returns nil or "" depend on length.
58
+ # ios.read() and ios.read(nil) returns "".
59
+ # ios.read(length [, buffer]) returns nil.
60
+ #
61
+ # If the Content-Length of the HTTP request is known (as is the common
62
+ # case for POST requests), then ios.read(length [, buffer]) will block
63
+ # until the specified length is read (or it is the last chunk).
64
+ # Otherwise, for uncommon "Transfer-Encoding: chunked" requests,
65
+ # ios.read(length [, buffer]) will return immediately if there is
66
+ # any data and only block when nothing is available (providing
67
+ # IO#readpartial semantics).
68
+ def read(*args)
69
+ @client ? tee(super) : @tmp.read(*args)
70
+ end
71
+
72
+ # :call-seq:
73
+ # ios.gets => string or nil
74
+ #
75
+ # Reads the next ``line'' from the I/O stream; lines are separated
76
+ # by the global record separator ($/, typically "\n"). A global
77
+ # record separator of nil reads the entire unread contents of ios.
78
+ # Returns nil if called at the end of file.
79
+ # This takes zero arguments for strict Rack::Lint compatibility,
80
+ # unlike IO#gets.
81
+ def gets
82
+ @client ? tee(super) : @tmp.gets
83
+ end
84
+
85
+ # :call-seq:
86
+ # ios.rewind => 0
87
+ #
88
+ # Positions the *ios* pointer to the beginning of input, returns
89
+ # the offset (zero) of the +ios+ pointer. Subsequent reads will
90
+ # start from the beginning of the previously-buffered input.
91
+ def rewind
92
+ return 0 if 0 == @tmp.size
93
+ consume! if @client
94
+ @tmp.rewind # Rack does not specify what the return value is here
95
+ end
96
+
97
+ # consumes the stream of the socket
98
+ def consume!
99
+ junk = ""
100
+ rsize = __rsize
101
+ nil while read(rsize, junk)
102
+ end
103
+
104
+ def tee(buffer)
105
+ if buffer && buffer.size > 0
106
+ @tmp.write(buffer)
107
+ end
108
+ buffer
109
+ end
110
+
111
+ def close # returns nil
112
+ @tmp = @tmp.close
113
+ end
114
+ end
@@ -0,0 +1,27 @@
1
+ # -*- encoding: binary -*-
2
+ # Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
3
+ # License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
4
+ require 'tmpdir'
5
+
6
+ # some versions of Ruby had a broken Tempfile which didn't work
7
+ # well with unlinked files. This one is much shorter, easier
8
+ # to understand, and slightly faster (no delegation).
9
+ class Yahns::TmpIO < File # :nodoc:
10
+
11
+ # creates and returns a new File object. The File is unlinked
12
+ # immediately, switched to binary mode, and userspace output
13
+ # buffering is disabled
14
+ def self.new
15
+ fp = begin
16
+ super("#{Dir.tmpdir}/#{rand}", RDWR|CREAT|EXCL, 0600)
17
+ rescue Errno::EEXIST
18
+ retry
19
+ end
20
+ unlink(fp.path)
21
+ fp.binmode
22
+ fp.sync = true
23
+ fp
24
+ end
25
+
26
+ alias discard close
27
+ end
@@ -0,0 +1,36 @@
1
+ # -*- encoding: binary -*-
2
+ # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> et. al.
3
+ # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
4
+ require_relative 'wbuf_common'
5
+
6
+ class Yahns::Wbuf # :nodoc:
7
+ include Yahns::WbufCommon
8
+
9
+ def initialize(body, persist)
10
+ @tmpio = Yahns::TmpIO.new
11
+ @sf_offset = @sf_count = 0
12
+ @wbuf_persist = persist # whether or not we keep the connection alive
13
+ @body = body
14
+ end
15
+
16
+ def wbuf_write(client, buf)
17
+ @sf_count += @tmpio.write(buf)
18
+ case rv = client.trysendfile(@tmpio, @sf_offset, @sf_count)
19
+ when Integer
20
+ @sf_count -= rv
21
+ @sf_offset += rv
22
+ when :wait_writable, :wait_readable
23
+ return rv
24
+ else
25
+ raise "BUG: #{rv.nil ? "EOF" : rv.inspect} on tmpio " \
26
+ "sf_offset=#@sf_offset sf_count=#@sf_count"
27
+ end while @sf_count > 0
28
+ nil
29
+ end
30
+
31
+ # called by last wbuf_flush
32
+ def wbuf_close(client)
33
+ @tmpio = @tmpio.close
34
+ wbuf_close_common(client)
35
+ end
36
+ end
@@ -0,0 +1,32 @@
1
+ # -*- encoding: binary -*-
2
+ # Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
3
+ # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
4
+ require 'sendfile'
5
+ module Yahns::WbufCommon # :nodoc:
6
+ # returns nil on success, :wait_*able when blocked
7
+ # currently, we rely on each thread having exclusive access to the
8
+ # client socket, so this is never called concurrently with wbuf_write
9
+ def wbuf_flush(client)
10
+ case rv = client.trysendfile(@tmpio, @sf_offset, @sf_count)
11
+ when Integer
12
+ return wbuf_close(client) if (@sf_count -= rv) == 0 # all sent!
13
+
14
+ @sf_offset += rv # keep going otherwise
15
+ when :wait_writable, :wait_readable
16
+ return rv
17
+ else
18
+ raise "BUG: #{rv.nil? ? "EOF" : rv.inspect} on tmpio=#{@tmpio.inspect} " \
19
+ "sf_offset=#@sf_offset sf_count=#@sf_count"
20
+ end while true
21
+ end
22
+
23
+ def wbuf_close_common(client)
24
+ @body.close if @body.respond_to?(:close)
25
+ if @wbuf_persist.respond_to?(:call) # hijack
26
+ @wbuf_persist.call(client)
27
+ :delete
28
+ else
29
+ @wbuf_persist # true or false or Yahns::StreamFile
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,58 @@
1
+ # -*- encoding: binary -*-
2
+ # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
3
+ # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
4
+ class Yahns::Worker # :nodoc:
5
+ attr_accessor :nr
6
+ attr_reader :to_io
7
+
8
+ def initialize(nr)
9
+ @nr = nr
10
+ @to_io, @wr = Kgio::Pipe.new
11
+ end
12
+
13
+ def atfork_child
14
+ @wr = @wr.close # nil @wr to save space in worker process
15
+ end
16
+
17
+ def atfork_parent
18
+ @to_io = @to_io.close
19
+ self
20
+ end
21
+
22
+ # used in the worker process.
23
+ # This causes the worker to gracefully exit if the master
24
+ # dies unexpectedly.
25
+ def yahns_step
26
+ @to_io.kgio_tryread(11) == nil and Process.kill(:QUIT, $$)
27
+ :wait_readable
28
+ end
29
+
30
+ # worker objects may be compared to just plain Integers
31
+ def ==(other_nr) # :nodoc:
32
+ @nr == other_nr
33
+ end
34
+
35
+ # Changes the worker process to the specified +user+ and +group+
36
+ # This is only intended to be called from within the worker
37
+ # process from the +after_fork+ hook. This should be called in
38
+ # the +after_fork+ hook after any privileged functions need to be
39
+ # run (e.g. to set per-worker CPU affinity, niceness, etc)
40
+ #
41
+ # Any and all errors raised within this method will be propagated
42
+ # directly back to the caller (usually the +after_fork+ hook.
43
+ # These errors commonly include ArgumentError for specifying an
44
+ # invalid user/group and Errno::EPERM for insufficient privileges
45
+ def user(user, group = nil)
46
+ # we do not protect the caller, checking Process.euid == 0 is
47
+ # insufficient because modern systems have fine-grained
48
+ # capabilities. Let the caller handle any and all errors.
49
+ uid = Etc.getpwnam(user).uid
50
+ gid = Etc.getgrnam(group).gid if group
51
+ Yahns::Log.chown_all(uid, gid)
52
+ if gid && Process.egid != gid
53
+ Process.initgroups(user, gid)
54
+ Process::GID.change_privilege(gid)
55
+ end
56
+ Process.euid != uid and Process::UID.change_privilege(uid)
57
+ end
58
+ end
@@ -0,0 +1,29 @@
1
+ # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
2
+ # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
3
+ #
4
+ # this works with the __covmerge method in test/helper.rb
5
+ # run this file after all tests are run
6
+
7
+ # load the merged dump data
8
+ res = Marshal.load(IO.binread("coverage.dump"))
9
+
10
+ # Dirty little text formatter. I tried simplecov but the default
11
+ # HTML+JS is unusable without a GUI (I hate GUIs :P) and it would've
12
+ # taken me longer to search the Internets to find a plain-text
13
+ # formatter I like...
14
+ res.keys.sort.each do |filename|
15
+ cov = res[filename]
16
+ puts "==> #{filename} <=="
17
+ File.readlines(filename).each_with_index do |line, i|
18
+ n = cov[i]
19
+ if n == 0 # BAD
20
+ print(" *** 0 #{line}")
21
+ elsif n
22
+ printf("% 7u %s", n, line)
23
+ elsif line =~ /\S/ # probably a line with just "end" in it
24
+ print(" #{line}")
25
+ else # blank line
26
+ print "\n" # don't output trailing whitespace on blank lines
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,115 @@
1
+ # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
2
+ # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
3
+ $stdout.sync = $stderr.sync = Thread.abort_on_exception = true
4
+ require 'thread'
5
+
6
+ # Global Test Lock, to protect:
7
+ # Process.wait*, Dir.chdir, ENV, trap, require, etc...
8
+ GTL = Mutex.new
9
+
10
+ # fork-aware coverage data gatherer, see also test/covshow.rb
11
+ if ENV["COVERAGE"]
12
+ require "coverage"
13
+ COVMATCH = %r{/lib/yahns\b.*rb\z}
14
+ COVTMP = File.open("coverage.dump", IO::CREAT|IO::RDWR)
15
+ COVTMP.binmode
16
+ COVTMP.sync = true
17
+
18
+ def __covmerge
19
+ res = Coverage.result
20
+
21
+ # we own this file (at least until somebody tries to use NFS :x)
22
+ COVTMP.flock(File::LOCK_EX)
23
+
24
+ COVTMP.rewind
25
+ prev = COVTMP.read
26
+ prev = prev.empty? ? {} : Marshal.load(prev)
27
+ res.each do |filename, counts|
28
+ # filter out stuff that's not in our project
29
+ COVMATCH =~ filename or next
30
+
31
+ merge = prev[filename] || []
32
+ merge = merge
33
+ counts.each_with_index do |count, i|
34
+ count or next
35
+ merge[i] = (merge[i] || 0) + count
36
+ end
37
+ prev[filename] = merge
38
+ end
39
+ COVTMP.rewind
40
+ COVTMP.truncate(0)
41
+ COVTMP.write(Marshal.dump(prev))
42
+ COVTMP.flock(File::LOCK_UN)
43
+ end
44
+
45
+ Coverage.start
46
+ at_exit { at_exit { __covmerge } }
47
+ end
48
+
49
+ gem 'minitest'
50
+ require 'minitest/autorun'
51
+ require "tempfile"
52
+
53
+ Testcase = begin
54
+ Minitest::Test # minitest 5
55
+ rescue NameError
56
+ Minitest::Unit::TestCase # minitest 4
57
+ end
58
+
59
+ FIFOS = []
60
+ def tmpfifo
61
+ tmp = Tempfile.new(%w(yahns-test .fifo))
62
+ path = tmp.path
63
+ tmp.close!
64
+ assert system(*%W(mkfifo #{path})), "mkfifo #{path}"
65
+
66
+ GTL.synchronize do
67
+ if FIFOS.empty?
68
+ at_exit do
69
+ FIFOS.each { |(pid,_path)| File.unlink(_path) if $$ == pid }
70
+ end
71
+ end
72
+ FIFOS << [ $$, path ]
73
+ end
74
+ path
75
+ end
76
+
77
+ require 'tmpdir'
78
+ class Dir
79
+ require 'fileutils'
80
+ def Dir.mktmpdir
81
+ begin
82
+ d = "#{Dir.tmpdir}/#$$.#{rand}"
83
+ Dir.mkdir(d)
84
+ rescue Errno::EEXIST
85
+ end while true
86
+ begin
87
+ yield d
88
+ ensure
89
+ FileUtils.remove_entry(d)
90
+ end
91
+ end
92
+ end unless Dir.respond_to?(:mktmpdir)
93
+
94
+ def tmpfile(*args)
95
+ tmp = Tempfile.new(*args)
96
+ tmp.sync = true
97
+ tmp.binmode
98
+ tmp
99
+ end
100
+
101
+ require 'io/wait'
102
+ # needed for Rubinius 2.0.0, we only use IO#nread in tests
103
+ class IO
104
+ # this ignores buffers
105
+ def nread
106
+ buf = "\0" * 8
107
+ ioctl(0x541B, buf)
108
+ buf.unpack("l_")[0]
109
+ end
110
+ end if ! IO.method_defined?(:nread) && RUBY_PLATFORM =~ /linux/
111
+
112
+ require 'yahns'
113
+
114
+ # needed for parallel (MT) tests)
115
+ require 'yahns/rack'
@@ -0,0 +1,65 @@
1
+ # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
2
+ # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
3
+ require_relative 'helper'
4
+ require 'timeout'
5
+ require 'socket'
6
+ require 'net/http'
7
+
8
+ module ServerHelper
9
+ def check_err(err = @err)
10
+ err = File.open(err.path, "r") if err.respond_to?(:path)
11
+ err.rewind
12
+ lines = err.readlines.delete_if { |l| l =~ /INFO/ }
13
+ assert lines.empty?, lines.join("\n")
14
+ err.close! if err == @err
15
+ end
16
+
17
+ def poke_until_dead(pid)
18
+ Timeout.timeout(10) do
19
+ begin
20
+ Process.kill(0, pid)
21
+ sleep(0.01)
22
+ rescue Errno::ESRCH
23
+ break
24
+ end while true
25
+ end
26
+ assert_raises(Errno::ESRCH) { Process.kill(0, pid) }
27
+ end
28
+
29
+ def quit_wait(pid)
30
+ pid or return
31
+ Process.kill(:QUIT, pid)
32
+ _, status = Timeout.timeout(10) { Process.waitpid2(pid) }
33
+ assert status.success?, status.inspect
34
+ rescue Timeout::Error
35
+ if RUBY_PLATFORM =~ /linux/
36
+ system("lsof -p #{pid}")
37
+ warn "#{pid} failed to die, waiting for user to inspect"
38
+ sleep
39
+ end
40
+ raise
41
+ end
42
+
43
+ # only use for newly bound sockets
44
+ def get_tcp_client(host, port, tries = 500)
45
+ begin
46
+ c = TCPSocket.new(host, port)
47
+ return c
48
+ rescue Errno::ECONNREFUSED
49
+ raise if tries < 0
50
+ tries -= 1
51
+ end while sleep(0.01)
52
+ end
53
+
54
+ def server_helper_teardown
55
+ @srv.close unless @srv.closed?
56
+ @ru.close! if @ru
57
+ check_err
58
+ end
59
+
60
+ def server_helper_setup
61
+ @srv = TCPServer.new(ENV["TEST_HOST"] || "127.0.0.1", 0)
62
+ @err = tmpfile(%w(srv .err))
63
+ @ru = nil
64
+ end
65
+ end