yahns 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Documentation/yahns_config.txt +20 -2
- data/GIT-VERSION-GEN +1 -1
- data/Rakefile +114 -38
- data/extras/exec_cgi.rb +22 -8
- data/lib/yahns/acceptor.rb +1 -1
- data/lib/yahns/daemon.rb +5 -1
- data/lib/yahns/fdmap.rb +12 -9
- data/lib/yahns/http_client.rb +2 -1
- data/lib/yahns/http_response.rb +13 -13
- data/lib/yahns/queue_epoll.rb +17 -4
- data/lib/yahns/server.rb +17 -6
- data/lib/yahns/stream_file.rb +1 -1
- data/lib/yahns/tmpio.rb +1 -1
- data/lib/yahns/wbuf.rb +24 -0
- data/lib/yahns/wbuf_str.rb +15 -1
- data/test/server_helper.rb +1 -0
- data/test/test_extras_exec_cgi.rb +104 -5
- data/test/test_extras_exec_cgi.sh +17 -0
- data/test/test_extras_try_gzip_static.rb +2 -1
- data/test/test_server.rb +0 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fff238b384ecd740912bd989b3f9f5940f1a8ed1
|
4
|
+
data.tar.gz: 4f0a23620f3de110285e6f0da568dbb8448713ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 019a14bdaf0613f75c89b1b170f705373698645a115f06f260e008080f3cb1b365091a2cfface0695bc841b090fd1d528d35b0ea54f0fa4b74424c6d28fc4cd0
|
7
|
+
data.tar.gz: ff28c0cee74dc2602ca0eb74dca63e9b98e84c28179f043db8875f29537b15a3d6df9b93439fbdc12cdfe6deb7006575248d7e54cce56351baefde378ba62494
|
data/.gitignore
CHANGED
@@ -278,6 +278,11 @@ Ruby it is running under.
|
|
278
278
|
|
279
279
|
HTTP request headers are always buffered in memory.
|
280
280
|
|
281
|
+
Do not be tempted to disable any buffering because it improves
|
282
|
+
numbers on a synthetic benchmark over a fast connection.
|
283
|
+
Slow, real-world clients can easily overwhelm servers without both
|
284
|
+
input and output buffering.
|
285
|
+
|
281
286
|
Default: true
|
282
287
|
|
283
288
|
The following OPTIONS may be specified:
|
@@ -418,12 +423,19 @@ Ruby it is running under.
|
|
418
423
|
|
419
424
|
This enables or disables buffering of the HTTP response. If enabled,
|
420
425
|
buffering is only performed lazily. In other words, buffering only
|
421
|
-
happens if socket buffers (in the kernel) are filled up
|
426
|
+
happens if socket buffers (in the kernel) are filled up, and yahns
|
427
|
+
will continously try to flush the buffered data to the socket while
|
428
|
+
it is buffering.
|
422
429
|
|
423
|
-
Disabling output buffering is only recommended if
|
430
|
+
Disabling output buffering is only recommended if ALL clients
|
424
431
|
connecting to this application context are fast, trusted or
|
425
432
|
you are willing and able to run many worker threads.
|
426
433
|
|
434
|
+
Do not be tempted to disable any buffering because it improves
|
435
|
+
numbers on a synthetic benchmark over a fast connection.
|
436
|
+
Slow, real-world clients can easily overwhelm servers without both
|
437
|
+
input and output buffering.
|
438
|
+
|
427
439
|
If output buffering is disabled, client_timeout controls the maximum
|
428
440
|
amount of time a worker thread will wait synchronously.
|
429
441
|
|
@@ -495,6 +507,12 @@ Ruby it is running under.
|
|
495
507
|
pthread_atfork(3)-style hooks. See WORKER_PROCESSES-LEVEL DIRECTIVES
|
496
508
|
for details.
|
497
509
|
|
510
|
+
Using worker_processes is strongly recommended if your application
|
511
|
+
relies on using a SIGCHLD handler for reaping forked processes.
|
512
|
+
Without worker_processes, yahns must reserve SIGCHLD for rolling
|
513
|
+
back SIGUSR2 upgrades, leading to conflicts if the appplication
|
514
|
+
expects to handle SIGCHLD.
|
515
|
+
|
498
516
|
Default: nil (single process, no master/worker separation)
|
499
517
|
|
500
518
|
## WORKER_PROCESSES-LEVEL DIRECTIVES
|
data/GIT-VERSION-GEN
CHANGED
data/Rakefile
CHANGED
@@ -2,59 +2,135 @@
|
|
2
2
|
# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
|
3
3
|
require 'tempfile'
|
4
4
|
include Rake::DSL
|
5
|
-
task "NEWS" do
|
6
|
-
latest = nil
|
7
|
-
fp = Tempfile.new("NEWS", ".")
|
8
|
-
fp.sync = true
|
9
|
-
`git tag -l`.split(/\n/).reverse.each do |tag|
|
10
|
-
%r{\Av(.+)} =~ tag or next
|
11
|
-
version = $1
|
12
|
-
header, subject, body = `git cat-file tag #{tag}`.split(/\n\n/, 3)
|
13
|
-
header = header.split(/\n/)
|
14
|
-
tagger = header.grep(/\Atagger /)[0]
|
15
|
-
time = Time.at(tagger.split(/ /)[-2].to_i).utc
|
16
|
-
latest ||= time
|
17
|
-
date = time.strftime("%Y-%m-%d")
|
18
|
-
fp.puts "# #{version} / #{date}\n\n#{subject}"
|
19
|
-
if body && body.strip.size > 0
|
20
|
-
fp.puts "\n\n#{body}"
|
21
|
-
end
|
22
|
-
fp.puts
|
23
|
-
end
|
24
|
-
fp.write("Unreleased\n\n") unless fp.size > 0
|
25
|
-
fp.puts "# COPYRIGHT"
|
26
|
-
bdfl = 'Eric Wong <normalperson@yhbt.net>'
|
27
|
-
fp.puts "Copyright (C) 2013, #{bdfl} and all contributors"
|
28
|
-
fp.puts "License: GPLv3 or later (http://www.gnu.org/licenses/gpl-3.0.txt)"
|
29
|
-
fp.rewind
|
30
|
-
assert_equal fp.read, File.read("NEWS") rescue nil
|
31
|
-
fp.chmod 0644
|
32
|
-
File.rename(fp.path, "NEWS")
|
33
|
-
fp.close!
|
34
|
-
end
|
35
5
|
|
36
|
-
|
6
|
+
gendocs = %w(NEWS NEWS.atom.xml)
|
7
|
+
task rsync_docs: gendocs do
|
37
8
|
dest = ENV["RSYNC_DEST"] || "yahns.yhbt.net:/srv/yahns/"
|
38
|
-
top = %w(INSTALL HACKING
|
39
|
-
files = []
|
9
|
+
top = %w(INSTALL HACKING README COPYING)
|
40
10
|
|
41
11
|
# git-set-file-times is distributed with rsync,
|
42
12
|
# Also available at: http://yhbt.net/git-set-file-times
|
43
13
|
# on Debian systems: /usr/share/doc/rsync/scripts/git-set-file-times.gz
|
44
14
|
sh("git", "set-file-times", "Documentation", "examples", *top)
|
45
15
|
|
46
|
-
|
16
|
+
do_gzip = lambda do |txt|
|
47
17
|
gz = "#{txt}.gz"
|
48
18
|
tmp = "#{gz}.#$$"
|
49
|
-
sh("gzip -9 < #{txt} > #{tmp}")
|
19
|
+
sh("gzip --rsyncable -9 < #{txt} > #{tmp}")
|
50
20
|
st = File.stat(txt)
|
51
21
|
File.utime(st.atime, st.mtime, tmp) # make nginx gzip_static happy
|
52
22
|
File.rename(tmp, gz)
|
53
|
-
|
54
|
-
files << gz
|
23
|
+
gz
|
55
24
|
end
|
25
|
+
|
26
|
+
files = `git ls-files Documentation/*.txt`.split(/\n/)
|
27
|
+
files.concat(top)
|
28
|
+
files.concat(gendocs)
|
29
|
+
gzfiles = files.map { |txt| do_gzip.call(txt) }
|
30
|
+
files.concat(gzfiles)
|
31
|
+
|
56
32
|
sh("rsync --chmod=Fugo=r -av #{files.join(' ')} #{dest}")
|
57
33
|
|
58
|
-
examples = `git ls-files examples`.split(
|
34
|
+
examples = `git ls-files examples`.split(/\n/)
|
35
|
+
gzex = examples.map { |txt| do_gzip.call(txt) }
|
36
|
+
examples.concat(gzex)
|
37
|
+
|
59
38
|
sh("rsync --chmod=Fugo=r -av #{examples.join(' ')} #{dest}/examples/")
|
60
39
|
end
|
40
|
+
|
41
|
+
def tags
|
42
|
+
timefmt = '%Y-%m-%dT%H:%M:%SZ'
|
43
|
+
@tags ||= `git tag -l`.split(/\n/).map do |tag|
|
44
|
+
if %r{\Av[\d\.]+} =~ tag
|
45
|
+
header, subject, body = `git cat-file tag #{tag}`.split(/\n\n/, 3)
|
46
|
+
header = header.split(/\n/)
|
47
|
+
tagger = header.grep(/\Atagger /).first
|
48
|
+
body ||= "initial"
|
49
|
+
time = Time.at(tagger.split(/ /)[-2].to_i).utc
|
50
|
+
{
|
51
|
+
time_obj: time,
|
52
|
+
time: time.strftime(timefmt),
|
53
|
+
tagger_name: %r{^tagger ([^<]+)}.match(tagger)[1].strip,
|
54
|
+
tagger_email: %r{<([^>]+)>}.match(tagger)[1].strip,
|
55
|
+
id: `git rev-parse refs/tags/#{tag}`.chomp!,
|
56
|
+
tag: tag,
|
57
|
+
subject: subject,
|
58
|
+
body: body,
|
59
|
+
}
|
60
|
+
end
|
61
|
+
end.compact.sort { |a,b| b[:time] <=> a[:time] }
|
62
|
+
end
|
63
|
+
|
64
|
+
url_base = 'http://yahns.yhbt.net'
|
65
|
+
cgit_url = 'http://yhbt.net/yahns.git'
|
66
|
+
git_url = 'git://yhbt.net/yahns.git'
|
67
|
+
since = "v0.0.2"
|
68
|
+
|
69
|
+
desc 'prints news as an Atom feed'
|
70
|
+
task "NEWS.atom.xml" do
|
71
|
+
require 'nokogiri'
|
72
|
+
new_tags = tags[0,10]
|
73
|
+
time = nil
|
74
|
+
str = Nokogiri::XML::Builder.new do
|
75
|
+
feed :xmlns => "http://www.w3.org/2005/Atom" do
|
76
|
+
id! "#{url_base}/NEWS.atom.xml"
|
77
|
+
title "yahns news"
|
78
|
+
subtitle "sleepy, multi-threaded, non-blocking Ruby application server"
|
79
|
+
link! :rel => 'alternate', :type => 'text/plain',
|
80
|
+
:href => "#{url_base}/NEWS"
|
81
|
+
updated(new_tags.empty? ? "1970-01-01T00:00:00Z" : new_tags.first[:time])
|
82
|
+
new_tags.each do |tag|
|
83
|
+
time ||= tag[:time_obj]
|
84
|
+
entry do
|
85
|
+
title tag[:subject]
|
86
|
+
updated tag[:time]
|
87
|
+
published tag[:time]
|
88
|
+
author {
|
89
|
+
name tag[:tagger_name]
|
90
|
+
email tag[:tagger_email]
|
91
|
+
}
|
92
|
+
url = "#{cgit_url}/tag/?id=#{tag[:tag]}"
|
93
|
+
link! :rel => "alternate", :type => "text/html", :href =>url
|
94
|
+
id! url
|
95
|
+
message_only = tag[:body].split(/\n.+\(\d+\):\n {6}/).first.strip
|
96
|
+
content({:type =>:text}, message_only)
|
97
|
+
content(:type =>:xhtml) { pre tag[:body] }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end.to_xml
|
102
|
+
|
103
|
+
fp = Tempfile.new("NEWS.atom.xml", ".")
|
104
|
+
fp.sync = true
|
105
|
+
fp.write(str)
|
106
|
+
fp.chmod 0644
|
107
|
+
File.utime(time, time, fp.path) if time
|
108
|
+
File.rename(fp.path, "NEWS.atom.xml")
|
109
|
+
fp.close!
|
110
|
+
end
|
111
|
+
|
112
|
+
desc 'prints news as a text file'
|
113
|
+
task "NEWS" do
|
114
|
+
fp = Tempfile.new("NEWS", ".")
|
115
|
+
fp.sync = true
|
116
|
+
time = nil
|
117
|
+
tags.each do |tag|
|
118
|
+
time ||= tag[:time_obj]
|
119
|
+
line = tag[:subject] + " / " + tag[:time].gsub(/T.*/, '')
|
120
|
+
fp.puts line
|
121
|
+
fp.puts("-" * line.length)
|
122
|
+
fp.puts
|
123
|
+
fp.puts tag[:body] #.gsub(/^/m, " ").gsub(/[ \t]+$/m, "")
|
124
|
+
fp.puts
|
125
|
+
end
|
126
|
+
fp.write("Unreleased\n\n") unless fp.size > 0
|
127
|
+
fp.puts "COPYRIGHT"
|
128
|
+
fp.puts "---------"
|
129
|
+
bdfl = 'Eric Wong <normalperson@yhbt.net>'
|
130
|
+
fp.puts "Copyright (C) 2013, #{bdfl} and all contributors"
|
131
|
+
fp.puts "License: GPLv3 or later (http://www.gnu.org/licenses/gpl-3.0.txt)"
|
132
|
+
fp.chmod 0644
|
133
|
+
File.utime(time, time, fp.path) if time
|
134
|
+
File.rename(fp.path, "NEWS")
|
135
|
+
fp.close!
|
136
|
+
end
|
data/extras/exec_cgi.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
|
3
3
|
# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
|
4
|
+
#
|
5
|
+
# if running under yahns, worker_processes is recommended to avoid conflicting
|
6
|
+
# with the SIGCHLD handler in yahns.
|
4
7
|
class ExecCgi
|
5
8
|
class MyIO < Kgio::Pipe
|
6
9
|
attr_writer :my_pid
|
@@ -19,17 +22,24 @@ class ExecCgi
|
|
19
22
|
end
|
20
23
|
yield("0\r\n\r\n") if @chunked
|
21
24
|
self
|
25
|
+
ensure
|
26
|
+
# do this sooner, since the response body may be buffered, we want
|
27
|
+
# to release our FD as soon as possible.
|
28
|
+
close
|
22
29
|
end
|
23
30
|
|
24
31
|
def close
|
32
|
+
# yahns will call this again after its done writing the response
|
33
|
+
# body, so we must ensure its idempotent.
|
34
|
+
# Note: this object (and any client-specific objects) will never
|
35
|
+
# be shared across different threads, so we do not need extra
|
36
|
+
# mutual exclusion here.
|
37
|
+
return if closed?
|
25
38
|
super
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
end
|
31
|
-
end
|
32
|
-
nil
|
39
|
+
begin
|
40
|
+
Process.waitpid(@my_pid)
|
41
|
+
rescue Errno::ECHILD
|
42
|
+
end if defined?(@my_pid) && @my_pid
|
33
43
|
end
|
34
44
|
end
|
35
45
|
|
@@ -67,7 +77,8 @@ class ExecCgi
|
|
67
77
|
PASS_VARS.each { |key| val = env[key] and cgi_env[key] = val }
|
68
78
|
env.each { |key,val| cgi_env[key] = val if key =~ /\AHTTP_/ }
|
69
79
|
pipe = MyIO.pipe
|
70
|
-
pipe[0]
|
80
|
+
errbody = pipe[0]
|
81
|
+
errbody.my_pid = Process.spawn(cgi_env, *@args,
|
71
82
|
out: pipe[1], close_others: true)
|
72
83
|
pipe[1].close
|
73
84
|
pipe = pipe[0]
|
@@ -100,9 +111,12 @@ class ExecCgi
|
|
100
111
|
pipe.chunked = true
|
101
112
|
end
|
102
113
|
end
|
114
|
+
errbody = nil
|
103
115
|
[ status, headers, pipe ]
|
104
116
|
else
|
105
117
|
[ 500, { "Content-Length" => "0", "Content-Type" => "text/plain" }, [] ]
|
106
118
|
end
|
119
|
+
ensure
|
120
|
+
errbody.close if errbody
|
107
121
|
end
|
108
122
|
end
|
data/lib/yahns/acceptor.rb
CHANGED
@@ -56,7 +56,7 @@ module Yahns::Acceptor # :nodoc:
|
|
56
56
|
end
|
57
57
|
rescue Errno::EMFILE, Errno::ENFILE => e
|
58
58
|
logger.error("#{e.message}, consider raising open file limits")
|
59
|
-
queue.fdmap.
|
59
|
+
queue.fdmap.desperate_expire(5)
|
60
60
|
sleep 1 # let other threads do some work
|
61
61
|
rescue => e
|
62
62
|
Yahns::Log.exception(logger, "accept loop", e)
|
data/lib/yahns/daemon.rb
CHANGED
@@ -19,7 +19,11 @@ module Yahns::Daemon # :nodoc:
|
|
19
19
|
|
20
20
|
# We only start a new process group if we're not being reexecuted
|
21
21
|
# and inheriting file descriptors from our parent
|
22
|
-
|
22
|
+
if ENV['YAHNS_FD']
|
23
|
+
# if we're inheriting, need to ensure this remains true so
|
24
|
+
# SIGWINCH works when worker processes are in play
|
25
|
+
yahns_server.daemon_pipe = true
|
26
|
+
else
|
23
27
|
# grandparent - reads pipe, exits when master is ready
|
24
28
|
# \_ parent - exits immediately ASAP
|
25
29
|
# \_ yahns master - writes to pipe when ready
|
data/lib/yahns/fdmap.rb
CHANGED
@@ -31,11 +31,16 @@ class Yahns::Fdmap # :nodoc:
|
|
31
31
|
|
32
32
|
# Yes, we call IO#close inside the lock(!)
|
33
33
|
#
|
34
|
-
# We don't want to race with
|
34
|
+
# We don't want to race with __expire. Theoretically, a Ruby
|
35
35
|
# implementation w/o GVL may end up issuing shutdown(2) on the same fd
|
36
36
|
# as one which got accept-ed (a brand new IO object) so we must prevent
|
37
37
|
# IO#close in worker threads from racing with any threads which may run
|
38
|
-
#
|
38
|
+
# __expire
|
39
|
+
#
|
40
|
+
# We must never, ever call this while it is capable of being on the
|
41
|
+
# epoll ready list and returnable by epoll_wait. So we can only call
|
42
|
+
# this on objects which were epoll_ctl-ed with EPOLLONESHOT (and now
|
43
|
+
# retrieved).
|
39
44
|
def sync_close(io)
|
40
45
|
@fdmap_mtx.synchronize do
|
41
46
|
@count -= 1
|
@@ -48,17 +53,16 @@ class Yahns::Fdmap # :nodoc:
|
|
48
53
|
fd = io.fileno
|
49
54
|
@fdmap_mtx.synchronize do
|
50
55
|
if (@count += 1) > @client_expire_threshold
|
51
|
-
|
52
|
-
else
|
53
|
-
@fdmap_ary[fd] = io
|
56
|
+
__expire(nil)
|
54
57
|
end
|
58
|
+
@fdmap_ary[fd] = io
|
55
59
|
end
|
56
60
|
end
|
57
61
|
|
58
62
|
# this is only called in Errno::EMFILE/Errno::ENFILE situations
|
59
63
|
# and graceful shutdown
|
60
|
-
def
|
61
|
-
@fdmap_mtx.synchronize {
|
64
|
+
def desperate_expire(timeout)
|
65
|
+
@fdmap_mtx.synchronize { __expire(timeout) }
|
62
66
|
end
|
63
67
|
|
64
68
|
# only called on hijack
|
@@ -74,7 +78,7 @@ class Yahns::Fdmap # :nodoc:
|
|
74
78
|
# expire a bunch of idle clients and register the current one
|
75
79
|
# We should not be calling this too frequently, it is expensive
|
76
80
|
# This is called while @fdmap_mtx is held
|
77
|
-
def
|
81
|
+
def __expire(timeout)
|
78
82
|
nr = 0
|
79
83
|
now = Time.now.to_f
|
80
84
|
(now - @last_expire) >= 1.0 or return # don't expire too frequently
|
@@ -88,7 +92,6 @@ class Yahns::Fdmap # :nodoc:
|
|
88
92
|
nr += c.yahns_expire(tout)
|
89
93
|
end
|
90
94
|
|
91
|
-
@fdmap_ary[io.fileno] = io if io
|
92
95
|
@last_expire = Time.now.to_f
|
93
96
|
msg = timeout ? "timeout=#{timeout}" : "client_timeout"
|
94
97
|
@logger.info("dropping #{nr} of #@count clients for #{msg}")
|
data/lib/yahns/http_client.rb
CHANGED
@@ -269,7 +269,8 @@ class Yahns::HttpClient < Kgio::Socket # :nodoc:
|
|
269
269
|
# nil to ensure the socket is closed at the end of this function
|
270
270
|
def handle_error(e)
|
271
271
|
code = case e
|
272
|
-
when EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::ENOTCONN
|
272
|
+
when EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::ENOTCONN,
|
273
|
+
Errno::ETIMEDOUT
|
273
274
|
return # don't send response, drop the connection
|
274
275
|
when Yahns::ClientTimeout
|
275
276
|
408
|
data/lib/yahns/http_response.rb
CHANGED
@@ -50,20 +50,20 @@ module Yahns::HttpResponse # :nodoc:
|
|
50
50
|
wbuf = Yahns::Wbuf.new(body, alive, self.class.output_buffer_tmpdir)
|
51
51
|
rv = wbuf.wbuf_write(self, header)
|
52
52
|
body.each { |chunk| rv = wbuf.wbuf_write(self, chunk) } if body
|
53
|
-
wbuf_maybe(wbuf, rv
|
53
|
+
wbuf_maybe(wbuf, rv)
|
54
54
|
end
|
55
55
|
|
56
|
-
def wbuf_maybe(wbuf, rv
|
57
|
-
case rv #
|
58
|
-
when nil
|
59
|
-
case
|
60
|
-
when :ignore
|
61
|
-
@state =
|
56
|
+
def wbuf_maybe(wbuf, rv)
|
57
|
+
case rv # wbuf_write return value
|
58
|
+
when nil # all done
|
59
|
+
case rv = wbuf.wbuf_close(self)
|
60
|
+
when :ignore # hijacked
|
61
|
+
@state = rv
|
62
62
|
when Yahns::StreamFile
|
63
|
-
@state =
|
63
|
+
@state = rv
|
64
64
|
:wait_writable
|
65
65
|
when true, false
|
66
|
-
http_response_done(
|
66
|
+
http_response_done(rv)
|
67
67
|
end
|
68
68
|
else
|
69
69
|
@state = wbuf
|
@@ -81,9 +81,9 @@ module Yahns::HttpResponse # :nodoc:
|
|
81
81
|
:wait_readable
|
82
82
|
else
|
83
83
|
@state = :pipelined
|
84
|
-
#
|
85
|
-
#
|
86
|
-
:
|
84
|
+
# we shouldn't start processing the application again until we know
|
85
|
+
# the socket is writable for the response
|
86
|
+
:wait_writable
|
87
87
|
end
|
88
88
|
else
|
89
89
|
# shutdown is needed in case the app forked, we rescue here since
|
@@ -184,7 +184,7 @@ module Yahns::HttpResponse # :nodoc:
|
|
184
184
|
# (or :wait_readable for SSL) and hit Yahns::HttpClient#step_write
|
185
185
|
if wbuf
|
186
186
|
body = nil # ensure we do not close the body in ensure
|
187
|
-
wbuf_maybe(wbuf, rv
|
187
|
+
wbuf_maybe(wbuf, rv)
|
188
188
|
else
|
189
189
|
http_response_done(alive)
|
190
190
|
end
|
data/lib/yahns/queue_epoll.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
|
3
3
|
# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
|
4
|
+
#
|
5
|
+
# This is the dangerous, low-level epoll interface for sleepy_penguin
|
6
|
+
# It is safe as long as you're aware of all potential concurrency
|
7
|
+
# issues given multithreading, GC, and epoll itself.
|
4
8
|
class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
|
5
9
|
include SleepyPenguin
|
6
10
|
attr_accessor :fdmap # Yahns::Fdmap
|
@@ -9,7 +13,6 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
|
|
9
13
|
QEV_QUIT = Epoll::OUT # Level Trigger for QueueQuitter
|
10
14
|
QEV_RD = Epoll::IN | Epoll::ONESHOT
|
11
15
|
QEV_WR = Epoll::OUT | Epoll::ONESHOT
|
12
|
-
QEV_RDWR = QEV_RD | QEV_WR
|
13
16
|
|
14
17
|
def self.new
|
15
18
|
super(SleepyPenguin::Epoll::CLOEXEC)
|
@@ -18,6 +21,8 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
|
|
18
21
|
# for HTTP and HTTPS servers, we rely on the io writing to us, first
|
19
22
|
# flags: QEV_RD/QEV_WR (usually QEV_RD)
|
20
23
|
def queue_add(io, flags)
|
24
|
+
# order is very important here, this thread cannot do anything with
|
25
|
+
# io once we've issued epoll_ctl() because another thread may use it
|
21
26
|
@fdmap.add(io)
|
22
27
|
epoll_ctl(Epoll::CTL_ADD, io, flags)
|
23
28
|
end
|
@@ -28,9 +33,14 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
|
|
28
33
|
end
|
29
34
|
|
30
35
|
# use only before hijacking, once hijacked, io may be unusable to us
|
36
|
+
# It is not safe to call this unless it is an unarmed EPOLLONESHOT
|
37
|
+
# object.
|
31
38
|
def queue_del(io)
|
32
|
-
|
39
|
+
# order does not really matter here, however Epoll::CTL_DEL
|
40
|
+
# will free up ~200 bytes of unswappable kernel memory,
|
41
|
+
# so we call it first
|
33
42
|
epoll_ctl(Epoll::CTL_DEL, io, 0)
|
43
|
+
@fdmap.forget(io)
|
34
44
|
end
|
35
45
|
|
36
46
|
# returns an array of infinitely running threads
|
@@ -39,13 +49,15 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
|
|
39
49
|
thr_init
|
40
50
|
begin
|
41
51
|
epoll_wait(max_events) do |_, io| # don't care for flags for now
|
52
|
+
|
53
|
+
# Note: we absolutely must not do anything with io after
|
54
|
+
# we've called epoll_ctl on it, io is exclusive to this
|
55
|
+
# thread only until epoll_ctl is called on it.
|
42
56
|
case rv = io.yahns_step
|
43
57
|
when :wait_readable
|
44
58
|
epoll_ctl(Epoll::CTL_MOD, io, QEV_RD)
|
45
59
|
when :wait_writable
|
46
60
|
epoll_ctl(Epoll::CTL_MOD, io, QEV_WR)
|
47
|
-
when :wait_readwrite
|
48
|
-
epoll_ctl(Epoll::CTL_MOD, io, QEV_RDWR)
|
49
61
|
when :ignore # only used by rack.hijack
|
50
62
|
# we cannot call Epoll::CTL_DEL after hijacking, the hijacker
|
51
63
|
# may have already closed it Likewise, io.fileno is not
|
@@ -59,6 +71,7 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
|
|
59
71
|
end
|
60
72
|
end
|
61
73
|
rescue => e
|
74
|
+
break if closed? # can still happen due to shutdown_timeout
|
62
75
|
Yahns::Log.exception(logger, 'queue loop', e)
|
63
76
|
end while true
|
64
77
|
end
|
data/lib/yahns/server.rb
CHANGED
@@ -20,6 +20,7 @@ class Yahns::Server # :nodoc:
|
|
20
20
|
include Yahns::SocketHelper
|
21
21
|
|
22
22
|
def initialize(config)
|
23
|
+
@shutdown_expire = nil
|
23
24
|
@shutdown_timeout = nil
|
24
25
|
@reexec_pid = 0
|
25
26
|
@daemon_pipe = nil # writable IO or true
|
@@ -201,7 +202,7 @@ class Yahns::Server # :nodoc:
|
|
201
202
|
end
|
202
203
|
|
203
204
|
def daemon_ready
|
204
|
-
@daemon_pipe or return
|
205
|
+
@daemon_pipe.respond_to?(:syswrite) or return
|
205
206
|
@daemon_pipe.syswrite("#$$")
|
206
207
|
@daemon_pipe.close
|
207
208
|
@daemon_pipe = true # for SIGWINCH
|
@@ -379,7 +380,8 @@ class Yahns::Server # :nodoc:
|
|
379
380
|
|
380
381
|
def quit_enter(alive)
|
381
382
|
if alive
|
382
|
-
@logger.info("gracefully exiting shutdown_timeout
|
383
|
+
@logger.info("gracefully exiting shutdown_timeout=#@shutdown_timeout")
|
384
|
+
@shutdown_expire ||= Time.now + @shutdown_timeout + 1
|
383
385
|
else # drop connections immediately if signaled twice
|
384
386
|
@logger.info("graceful exit aborted, exiting immediately")
|
385
387
|
# we will still call any app-defined at_exit hooks here
|
@@ -406,17 +408,25 @@ class Yahns::Server # :nodoc:
|
|
406
408
|
@queues.each { |q| q.queue_add(quitter, Yahns::Queue::QEV_QUIT) }
|
407
409
|
|
408
410
|
# watch the monkey wrench destroy all the threads!
|
409
|
-
|
411
|
+
# Ugh, this may fail if we have dedicated threads trickling
|
412
|
+
# response bodies out (e.g. "tail -F") Oh well, have a timeout
|
413
|
+
begin
|
414
|
+
@wthr.delete_if { |t| t.join(0.01) }
|
415
|
+
end while @wthr[0] && Time.now <= @shutdown_expire
|
410
416
|
|
411
417
|
# cleanup, our job is done
|
412
418
|
@queues.each(&:close).clear
|
419
|
+
|
420
|
+
# we must not let quitter get GC-ed if we have any worker threads leftover
|
421
|
+
@wthr.each { |t| t[:yahns_quitter] = quitter }
|
422
|
+
|
413
423
|
quitter.close
|
414
424
|
rescue => e
|
415
425
|
Yahns::Log.exception(@logger, "quit finish", e)
|
416
426
|
ensure
|
417
427
|
if (@wthr.size + @listeners.size) > 0
|
418
|
-
@logger.
|
419
|
-
|
428
|
+
@logger.warn("still active wthr=#{@wthr.size} "\
|
429
|
+
"listeners=#{@listeners.size}")
|
420
430
|
end
|
421
431
|
end
|
422
432
|
|
@@ -452,7 +462,8 @@ class Yahns::Server # :nodoc:
|
|
452
462
|
|
453
463
|
def dropping(fdmap)
|
454
464
|
if drop_acceptors[0] || fdmap.size > 0
|
455
|
-
|
465
|
+
timeout = @shutdown_expire < Time.now ? -1 : @shutdown_timeout
|
466
|
+
fdmap.desperate_expire(timeout)
|
456
467
|
true
|
457
468
|
else
|
458
469
|
false
|
data/lib/yahns/stream_file.rb
CHANGED
data/lib/yahns/tmpio.rb
CHANGED
data/lib/yahns/wbuf.rb
CHANGED
@@ -3,6 +3,30 @@
|
|
3
3
|
# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
|
4
4
|
require_relative 'wbuf_common'
|
5
5
|
|
6
|
+
# This class is triggered whenever we need write buffering for clients
|
7
|
+
# reading responses slowly. Small responses which fit into kernel socket
|
8
|
+
# buffers do not trigger this. yahns will always attempt to write to kernel
|
9
|
+
# socket buffers first to avoid unnecessary copies in userspace.
|
10
|
+
#
|
11
|
+
# Thus, most data is into copied to the kernel only once, the kernel
|
12
|
+
# will perform zero-copy I/O from the page cache to the socket. The
|
13
|
+
# only time data may be copied twice is the initial write()/send()
|
14
|
+
# which triggers EAGAIN.
|
15
|
+
#
|
16
|
+
# We only buffer to the filesystem (note: likely not a disk, since this
|
17
|
+
# is short-lived). We let the sysadmin/kernel decide whether or not
|
18
|
+
# the data needs to hit disk or not.
|
19
|
+
#
|
20
|
+
# This avoids large allocations from malloc, potentially limiting
|
21
|
+
# fragmentation and keeping (common) smaller allocations fast.
|
22
|
+
# General purpose malloc implementations in the 64-bit era tend to avoid
|
23
|
+
# releasing memory back to the kernel, so large heap allocations are best
|
24
|
+
# avoided as the kernel has little chance to reclaim memory used for a
|
25
|
+
# temporary buffer.
|
26
|
+
#
|
27
|
+
# The biggest downside of this approach is it requires an FD, but yahns
|
28
|
+
# configurations are configured for many FDs anyways, so it's unlikely
|
29
|
+
# to be a scalability issue.
|
6
30
|
class Yahns::Wbuf # :nodoc:
|
7
31
|
include Yahns::WbufCommon
|
8
32
|
|
data/lib/yahns/wbuf_str.rb
CHANGED
@@ -3,12 +3,26 @@
|
|
3
3
|
# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
|
4
4
|
require_relative 'wbuf_common'
|
5
5
|
|
6
|
+
# we only use this for buffering the tiniest responses (which are already
|
7
|
+
# strings in memory and a handful of bytes).
|
8
|
+
#
|
9
|
+
# "HTTP", "/1.1 "
|
10
|
+
# "HTTP/1.1 100 Continue\r\n\r\n"
|
11
|
+
# "100 Continue\r\n\r\nHTTP/1.1 "
|
12
|
+
#
|
13
|
+
# This is very, very rarely triggered.
|
14
|
+
# 1) check_client_connection is enabled
|
15
|
+
# 2) the client sent an "Expect: 100-continue" header
|
16
|
+
#
|
17
|
+
# Most output buffering goes through
|
18
|
+
# the normal Yahns::Wbuf class which uses a temporary file as a buffer
|
19
|
+
# (suitable for sendfile())
|
6
20
|
class Yahns::WbufStr # :nodoc:
|
7
21
|
include Yahns::WbufCommon
|
8
22
|
|
9
23
|
def initialize(str, next_state)
|
10
24
|
@str = str
|
11
|
-
@next = next_state # :
|
25
|
+
@next = next_state # :ccc_done, :r100_done
|
12
26
|
end
|
13
27
|
|
14
28
|
def wbuf_flush(client)
|
data/test/server_helper.rb
CHANGED
@@ -2,23 +2,24 @@
|
|
2
2
|
# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
|
3
3
|
require_relative 'server_helper'
|
4
4
|
|
5
|
+
# note: we use worker_processes to avoid polling/excessive wakeup issues
|
6
|
+
# in the test. We recommend using worker_processes if using ExecCgi
|
5
7
|
class TestExtrasExecCGI < Testcase
|
6
8
|
ENV["N"].to_i > 1 and parallelize_me!
|
7
9
|
include ServerHelper
|
8
10
|
alias setup server_helper_setup
|
9
11
|
alias teardown server_helper_teardown
|
12
|
+
RUNME = "#{Dir.pwd}/test/test_extras_exec_cgi.sh"
|
10
13
|
|
11
14
|
def test_exec_cgi
|
12
15
|
err, cfg, host, port = @err, Yahns::Config.new, @srv.addr[3], @srv.addr[1]
|
13
|
-
|
14
|
-
assert File.executable?(runme), "run test in project root"
|
16
|
+
assert File.executable?(RUNME), "run test in project root"
|
15
17
|
pid = mkserver(cfg) do
|
16
18
|
require './extras/exec_cgi'
|
17
19
|
cfg.instance_eval do
|
18
|
-
app(:rack, ExecCgi.new(
|
19
|
-
listen "#{host}:#{port}"
|
20
|
-
end
|
20
|
+
app(:rack, ExecCgi.new(RUNME)) { listen "#{host}:#{port}" }
|
21
21
|
stderr_path err.path
|
22
|
+
worker_processes 1
|
22
23
|
end
|
23
24
|
end
|
24
25
|
|
@@ -75,7 +76,105 @@ class TestExtrasExecCGI < Testcase
|
|
75
76
|
assert_nil body
|
76
77
|
c.close
|
77
78
|
end
|
79
|
+
|
80
|
+
Timeout.timeout(30) do # pid of executable
|
81
|
+
c = get_tcp_client(host, port)
|
82
|
+
c.write "GET /pid HTTP/1.0\r\n\r\n"
|
83
|
+
head, body = c.read.split(/\r\n\r\n/, 2)
|
84
|
+
assert_match %r{200 OK}, head
|
85
|
+
assert_match %r{\A\d+\n\z}, body
|
86
|
+
exec_pid = body.to_i
|
87
|
+
c.close
|
88
|
+
poke_until_dead exec_pid
|
89
|
+
end
|
90
|
+
ensure
|
91
|
+
quit_wait(pid)
|
92
|
+
end
|
93
|
+
|
94
|
+
def test_cgi_died
|
95
|
+
err, cfg, host, port = @err, Yahns::Config.new, @srv.addr[3], @srv.addr[1]
|
96
|
+
pid = mkserver(cfg) do
|
97
|
+
require './extras/exec_cgi'
|
98
|
+
cfg.instance_eval do
|
99
|
+
app(:rack, ExecCgi.new(RUNME)) { listen "#{host}:#{port}" }
|
100
|
+
stderr_path err.path
|
101
|
+
worker_processes 1
|
102
|
+
end
|
103
|
+
end
|
104
|
+
exec_pid_tmp = tmpfile(%w(exec_cgi .pid))
|
105
|
+
c = get_tcp_client(host, port)
|
106
|
+
Timeout.timeout(20) do
|
107
|
+
c.write "GET /die HTTP/1.0\r\nX-PID-DEST: #{exec_pid_tmp.path}\r\n\r\n"
|
108
|
+
head, body = c.read.split(/\r\n\r\n/, 2)
|
109
|
+
assert_match %r{500 Internal Server Error}, head
|
110
|
+
assert_match "", body
|
111
|
+
exec_pid = exec_pid_tmp.read
|
112
|
+
assert_match %r{\A(\d+)\n\z}, exec_pid
|
113
|
+
poke_until_dead exec_pid.to_i
|
114
|
+
end
|
115
|
+
ensure
|
116
|
+
exec_pid_tmp.close! if exec_pid_tmp
|
117
|
+
quit_wait(pid)
|
118
|
+
end
|
119
|
+
|
120
|
+
[ 9, 10, 11 ].each do |rtype|
|
121
|
+
[ 1, 2, 3 ].each do |block_on|
|
122
|
+
define_method("test_block_on_block_on_#{block_on}_rtype_#{rtype}") do
|
123
|
+
_blocked_zombie([block_on], rtype)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def _blocked_zombie(block_on, rtype)
|
129
|
+
err, cfg, host, port = @err, Yahns::Config.new, @srv.addr[3], @srv.addr[1]
|
130
|
+
pid = mkserver(cfg) do
|
131
|
+
$_tw_blocked = 0
|
132
|
+
$_tw_block_on = block_on
|
133
|
+
Yahns::HttpClient.__send__(:include, TrywriteBlocked)
|
134
|
+
require './extras/exec_cgi'
|
135
|
+
cfg.instance_eval do
|
136
|
+
app(:rack, ExecCgi.new(RUNME)) { listen "#{host}:#{port}" }
|
137
|
+
stderr_path err.path
|
138
|
+
worker_processes 1
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
c = get_tcp_client(host, port)
|
143
|
+
Timeout.timeout(20) do
|
144
|
+
case rtype
|
145
|
+
when 9 # non-persistent (HTTP/0.9)
|
146
|
+
c.write "GET /pid\r\n\r\n"
|
147
|
+
body = c.read
|
148
|
+
assert_match %r{\A\d+\n\z}, body
|
149
|
+
exec_pid = body.to_i
|
150
|
+
poke_until_dead exec_pid
|
151
|
+
when 10 # non-persistent (HTTP/1.0)
|
152
|
+
c.write "GET /pid HTTP/1.0\r\n\r\n"
|
153
|
+
head, body = c.read.split(/\r\n\r\n/, 2)
|
154
|
+
assert_match %r{200 OK}, head
|
155
|
+
assert_match %r{\A\d+\n\z}, body
|
156
|
+
exec_pid = body.to_i
|
157
|
+
poke_until_dead exec_pid
|
158
|
+
when 11 # pid of executable, persistent
|
159
|
+
c.write "GET /pid HTTP/1.0\r\nConnection: keep-alive\r\n\r\n"
|
160
|
+
buf = ""
|
161
|
+
begin
|
162
|
+
buf << c.readpartial(666)
|
163
|
+
end until buf =~ /\r\n\r\n\d+\n/
|
164
|
+
head, body = buf.split(/\r\n\r\n/, 2)
|
165
|
+
assert_match %r{200 OK}, head
|
166
|
+
assert_match %r{\A\d+\n\z}, body
|
167
|
+
exec_pid = body.to_i
|
168
|
+
assert_raises(Errno::EAGAIN, IO::WaitReadable) { c.read_nonblock(666) }
|
169
|
+
poke_until_dead exec_pid
|
170
|
+
# still alive?
|
171
|
+
assert_raises(Errno::EAGAIN, IO::WaitReadable) { c.read_nonblock(666) }
|
172
|
+
else
|
173
|
+
raise "BUG in test, bad rtype"
|
174
|
+
end
|
175
|
+
end
|
78
176
|
ensure
|
177
|
+
c.close if c
|
79
178
|
quit_wait(pid)
|
80
179
|
end
|
81
180
|
end
|
@@ -20,6 +20,23 @@ case $PATH_INFO in
|
|
20
20
|
stdhead
|
21
21
|
env
|
22
22
|
;;
|
23
|
+
/pid)
|
24
|
+
stdhead
|
25
|
+
echo $$
|
26
|
+
;;
|
27
|
+
/die)
|
28
|
+
if test -n "$HTTP_X_PID_DEST"
|
29
|
+
then
|
30
|
+
# obviously this is only for testing on a local machine:
|
31
|
+
echo $$ > "$HTTP_X_PID_DEST"
|
32
|
+
exit 1
|
33
|
+
else
|
34
|
+
echo Content-Type: text/plain
|
35
|
+
echo Status: 400 Bad Request
|
36
|
+
echo Content-Length: 0
|
37
|
+
echo
|
38
|
+
fi
|
39
|
+
;;
|
23
40
|
/known-length)
|
24
41
|
echo Content-Type: text/plain
|
25
42
|
echo Status: 200 OK
|
data/test/test_server.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yahns
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yahns hackers
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-11-
|
11
|
+
date: 2013-11-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: kgio
|