yahns 0.0.1 → 0.0.2

data/extras/exec_cgi.rb

@@ -0,0 +1,108 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+class ExecCgi
+  class MyIO < Kgio::Pipe
+    attr_writer :my_pid
+    attr_writer :body_tip
+    attr_writer :chunked
+
+    def each
+      buf = @body_tip || ""
+      if buf.size > 0
+        buf = "#{buf.size.to_s(16)}\r\n#{buf}\r\n" if @chunked
+        yield buf
+      end
+      while tmp = kgio_read(8192, buf)
+        tmp = "#{tmp.size.to_s(16)}\r\n#{tmp}\r\n" if @chunked
+        yield tmp
+      end
+      yield("0\r\n\r\n") if @chunked
+      self
+    end
+
+    def close
+      super
+      if defined?(@my_pid) && @my_pid
+        begin
+          Process.waitpid(@my_pid)
+        rescue Errno::ECHILD
+        end
+      end
+      nil
+    end
+  end
+
+  PASS_VARS = %w(
+    CONTENT_LENGTH
+    CONTENT_TYPE
+    GATEWAY_INTERFACE
+    AUTH_TYPE
+    PATH_INFO
+    PATH_TRANSLATED
+    QUERY_STRING
+    REMOTE_ADDR
+    REMOTE_HOST
+    REMOTE_IDENT
+    REMOTE_USER
+    REQUEST_METHOD
+    SERVER_NAME
+    SERVER_PORT
+    SERVER_PROTOCOL
+    SERVER_SOFTWARE
+  ).map(&:freeze)  # frozen strings are faster for Hash assignments
+
+  def initialize(*args)
+    @args = args
+    first = args[0] or
+      raise ArgumentError, "need path to executable"
+    first[0] == ?/ or args[0] = ::File.expand_path(first)
+    File.executable?(args[0]) or
+      raise ArgumentError, "#{args[0]} is not executable"
+  end
+
+  # Calls the app
+  def call(env)
+    cgi_env = { "SCRIPT_NAME" => @args[0], "GATEWAY_INTERFACE" => "CGI/1.1" }
+    PASS_VARS.each { |key| val = env[key] and cgi_env[key] = val }
+    env.each { |key,val| cgi_env[key] = val if key =~ /\AHTTP_/ }
+    pipe = MyIO.pipe
+    pipe[0].my_pid = Process.spawn(cgi_env, *@args,
+                                   out: pipe[1], close_others: true)
+    pipe[1].close
+    pipe = pipe[0]
+
+    if head = pipe.kgio_read(8192)
+      until head =~ /\r?\n\r?\n/
+        tmp = pipe.kgio_read(8192) or break
+        head << tmp
+      end
+      head, body = head.split(/\r?\n\r?\n/)
+      pipe.body_tip = body
+      pipe.chunked = false
+
+      headers = Rack::Utils::HeaderHash.new
+      prev = nil
+      head.split(/\r?\n/).each do |line|
+        case line
+        when /^([A-Za-z0-9-]+):\s*(.*)$/ then headers[prev = $1] = $2
+        when /^[ \t]/ then headers[prev] << "\n#{line}" if prev
+        end
+      end
+      status = headers.delete("Status") || 200
+      unless headers.include?("Content-Length") ||
+             headers.include?("Transfer-Encoding")
+        case env['HTTP_VERSION']
+        when 'HTTP/1.0', nil
+          # server will drop connection anyways
+        else
+          headers["Transfer-Encoding"] = "chunked"
+          pipe.chunked = true
+        end
+      end
+      [ status, headers, pipe ]
+    else
+      [ 500, { "Content-Length" => "0", "Content-Type" => "text/plain" }, [] ]
+    end
+  end
+end

data/extras/proxy_pass.rb

@@ -0,0 +1,210 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require 'time'
+require 'socket'
+require 'kgio'
+require 'kcar' # gem install kcar
+require 'rack/request'
+require 'thread'
+require 'timeout'
+
+# Totally synchronous and Rack 1.1-compatible, this will probably be rewritten.
+# to take advantage of rack.hijack and use the non-blocking I/O facilities
+# in yahns.  yahns may have to grow a supported API for that...
+# For now, we this blocks a worker thread; fortunately threads are reasonably
+# cheap on GNU/Linux...
+# This is totally untested but currently doesn't serve anything important.
+class ProxyPass # :nodoc:
+  CHUNK_SIZE = 16384
+  ERROR_502 = [ 502, {'Content-Length'=>'0','Content-Type'=>'text/plain'}, [] ]
+
+  class ConnPool
+    def initialize
+      @mtx = Mutex.new
+      @objs = []
+    end
+
+    def get
+      @mtx.synchronize { @objs.pop }
+    end
+
+    def put(obj)
+      @mtx.synchronize { @objs << obj }
+    end
+  end
+
+  class UpstreamSocket < Kgio::Socket # :nodoc:
+    attr_writer :expiry
+
+    # called automatically by kgio_read!
+    def kgio_wait_readable(timeout = nil)
+      super(timeout || wait_time)
+    end
+
+    def wait_time
+      tout = @expiry ? @expiry - Time.now : @timeout
+      raise Timeout::Error, "request timed out", [] if tout < 0
+      tout
+    end
+
+    def readpartial(bytes, buf = Thread.current[:proxy_pass_buf] ||= "")
+      case rv = kgio_read!(bytes, buf)
+      when String
+        @expiry += @timeout # bump expiry when we succeed
+      end
+      rv
+    end
+
+    def req_write(buf, timeout)
+      @timeout = timeout
+      @expiry = Time.now + timeout
+      case rv = kgio_trywrite(buf)
+      when :wait_writable
+        kgio_wait_writable(wait_time)
+      when nil
+        return
+      when String
+        buf = rv
+      end while true
+    end
+  end # class UpstreamSocket
+
+  class UpstreamResponse < Kcar::Response # :nodoc:
+    # Called by the Rack server at the end of a successful response
+    def close
+      reusable = @parser.keepalive? && @parser.body_eof?
+      super
+      @pool.put(self) if reusable
+      nil
+    end
+
+    # req is just a string buffer of HTTP headers
+    def req_write(req, timeout)
+      @sock.req_write(req, timeout)
+    end
+
+    # returns true if the socket is still alive, nil if dead
+    def sock_alive?
+      @reused = (:wait_readable == (@sock.kgio_tryread(1) rescue nil)) ?
+                true : @sock.close
+    end
+
+    # returns true if the socket was reused and thus retryable
+    def fail_retryable?
+      @sock.close
+      @reused
+    end
+
+    def initialize(sock, pool)
+      super(sock)
+      @reused = false
+      @pool = pool
+    end
+  end # class UpstreamResponse
+
+  # take a responder from the pool, we'll add the object back to the
+  # pool in UpstreamResponse#close
+  def responder_get
+    while obj = @pool.get
+      return obj if obj.sock_alive?
+    end
+
+    UpstreamResponse.new(UpstreamSocket.start(@sockaddr), @pool)
+  end
+
+  def initialize(dest, timeout = 5)
+    case dest
+    when %r{\Ahttp://([^/]+)(/.*)\z}
+      path = $2
+      host, port = $1.split(/:/)
+      @sockaddr = Socket.sockaddr_in(port || 80, host)
+
+      # methods from Rack::Request we want:
+      allow = %w(fullpath host_with_port host port url path)
+      @path = path
+      want = path.scan(/\$(\w+)/).flatten! || []
+      diff = want - allow
+      diff.empty? or
+               raise ArgumentError, "vars not allowed: #{diff.uniq.join(' ')}"
+    else
+      raise ArgumentError, "destination must be an HTTP URL"
+    end
+    @pool = ConnPool.new
+    @timeout = timeout
+  end
+
+  def call(env)
+    case request_method = env["REQUEST_METHOD"]
+    when "GET", "HEAD" # OK
+    else
+      return [ 405, [%w(Content-Length 0), %w(Content-Length 0)], [] ]
+    end
+
+    req = Rack::Request.new(env)
+    path = @path.gsub(/\$(\w+)/) { req.__send__($1.to_sym) }
+    req = "#{request_method} #{path} HTTP/1.1\r\n" \
+          "X-Forwarded-For: #{env["REMOTE_ADDR"]}\r\n"
+
+    # pass most HTTP_* headers through as-is
+    chunked = false
+    env.each do |key, val|
+      %r{\AHTTP_(\w+)\z} =~ key or next
+      key = $1
+      next if %r{\A(?:VERSION|CONNECTION|KEEP_ALIVE|X_FORWARDED_FOR)} =~ key
+      chunked = true if %r{\ATRANSFER_ENCODING} =~ key && val =~ /\bchunked\b/i
+      key.tr!("_", "-")
+      req << "#{key}: #{val}\r\n"
+    end
+
+    # special cases which Rack does not prefix:
+    ctype = env["CONTENT_TYPE"] and req << "Content-Type: #{ctype}\r\n"
+    clen = env["CONTENT_LENGTH"] and req << "Content-Length: #{clen}\r\n"
+    req << "\r\n"
+
+    # get an open socket and send the headers
+    ures = responder_get
+    ures.req_write(req, @timeout)
+
+    # send the request body if there was one
+    send_body(env["rack.input"], ures, chunked) if chunked || clen
+
+    # wait for the response here
+    status, header, body = res = ures.rack
+
+    # don't let the upstream Connection and Keep-Alive headers leak through
+    header.delete_if do |k,_|
+      k =~ /\A(?:Connection|Keep-Alive)\z/i
+    end
+
+    case request_method
+    when "HEAD"
+      # kcar doesn't know if it's a HEAD or GET response, and HEAD
+      # responses have Content-Length in it which fools kcar...
+      body.parser.body_bytes_left = 0
+      res[1] = header.dup
+      body.close # clobbers original header
+      res[2] = body = []
+    end
+    res
+  rescue => e
+    retry if ures && ures.fail_retryable? && request_method != "POST"
+    ERROR_502
+  end
+
+  def send_body(input, ures, chunked)
+    buf = Thread.current[:proxy_pass_buf] ||= ""
+
+    if chunked # unlikely
+      while input.read(16384, buf)
+        buf.replace("#{buf.size.to_s(16)}\r\n#{buf}\r\n")
+        ures.req_write(buf, @timeout)
+      end
+      ures.req_write("0\r\n\r\n")
+    else # common if we hit uploads
+      while input.read(16384, buf)
+        ures.req_write(buf, @timeout)
+      end
+    end
+  end
+end

data/extras/try_gzip_static.rb

@@ -0,0 +1,208 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require 'time'
+require 'rack/utils'
+require 'rack/mime'
+require 'kgio'
+
+class TryGzipStatic
+  attr_accessor :root
+  class KF < Kgio::File
+    # attr_writer :sf_range
+
+    # only used if the server does not handle #to_path,
+    # yahns should never hit this
+    def each
+      raise "we should never get here in yahns"
+      buf = ""
+      rsize = 8192
+      if @sf_range
+        file.seek(@sf_range.begin)
+        sf_count = @sf_range.end - @sf_range.begin + 1
+        while sf_count > 0
+          read(sf_count > rsize ? rsize : sf_count, buf) or break
+          sf_count -= buf.size
+          yield buf
+        end
+        raise "file truncated" if sf_count != 0
+      else
+        yield(buf) while read(rsize, buf)
+      end
+    end
+  end
+
+  def initialize(root, default_type = 'text/plain')
+    @root = root
+    @default_type = default_type
+  end
+
+  def fspath(env)
+    path_info = Rack::Utils.unescape(env["PATH_INFO"])
+    path_info =~ /\.\./ ? nil : "#@root#{path_info}"
+  end
+
+  def get_range(env, path, st)
+    if ims = env["HTTP_IF_MODIFIED_SINCE"]
+      return [ 304, {}, [] ] if st.mtime.httpdate == ims
+    end
+
+    size = st.size
+    ranges = Rack::Utils.byte_ranges(env, size)
+    if ranges.nil? || ranges.length > 1
+      [ 200 ] # serve the whole thing, possibly with static gzip \o/
+    elsif ranges.empty?
+      res = r(416)
+      res[1]["Content-Range"] = "bytes */#{size}"
+      res
+    else # partial response, no using static gzip file
+      range = ranges[0]
+      len = range.end - range.begin + 1
+      h = fheader(env, path, st, nil, len)
+      h["Content-Range"] = "bytes #{range.begin}-#{range.end}/#{size}"
+      [ 206, h, range ]
+    end
+  end
+
+  def fheader(env, path, st, gz_st = nil, len = nil)
+    if path =~ /(.[^.]+)\z/
+      mime = Rack::Mime.mime_type($1, @default_type)
+    else
+      mime = @default_type
+    end
+    len ||= (gz_st ? gz_st : st).size
+    h = {
+      "Content-Type" => mime,
+      "Content-Length" => len.to_s,
+      "Last-Modified" => st.mtime.httpdate,
+      "Accept-Ranges" => "bytes",
+    }
+    h["Content-Encoding"] = "gzip" if gz_st
+    h
+  end
+
+  def head_no_gz(res, env, path, st)
+    res[1] = fheader(env, path, st)
+    res[2] = [] # empty body
+    res
+  end
+
+  def stat_path(env)
+    path = fspath(env) or return r(403)
+    begin
+      st = File.stat(path)
+      st.file? ? [ path, st ] : r(404)
+    rescue Errno::ENOENT
+      r(404)
+    rescue Errno::EACCES
+      r(403)
+    rescue => e
+      r(500, e.message, env)
+    end
+  end
+
+  def head(env)
+    path, st = res = stat_path(env)
+    return res if Integer === path # integer status code on failure
+
+    # see if it's a range request, no gzipped version if so
+    status, _ = res = get_range(env, path, st)
+    case status
+    when 206
+      res[2] = [] # empty body, headers  are all set
+      res
+    when 200 # fall through to trying gzipped version
+      # client requested gzipped path explicitly or did not want gzip
+      if path =~ /\.gz\z/i || !want_gzip?(env)
+        head_no_gz(res, env, path, st)
+      else # try the gzipped version
+        begin
+          gz_st = File.stat("#{path}.gz")
+          if gz_st.mtime == st.mtime
+            res[1] = fheader(env, path, st, gz_st)
+            res[2] = []
+            res
+          else
+            head_no_gz(res, env, path, st)
+          end
+        rescue Errno::ENOENT, Errno::EACCES
+          head_no_gz(res, env, path, st)
+        rescue => e
+          r(500, e.message, env)
+        end
+      end
+    else # 416, 304
+      res
+    end
+  end
+
+  def call(env)
+    case env["REQUEST_METHOD"]
+    when "GET" then get(env)
+    when "HEAD" then head(env)
+    else r(405)
+    end
+  end
+
+  def want_gzip?(env)
+    env["HTTP_ACCEPT_ENCODING"] =~ /\bgzip\b/i
+  end
+
+  def get(env)
+    path, st, _ = res = stat_path(env)
+    return res if Integer === path # integer status code on failure
+
+    # see if it's a range request, no gzipped version if so
+    status, _, _ = res = get_range(env, path, st)
+    case status
+    when 206
+      res[2] = KF.open(path) # stat succeeded
+    when 200
+      # client requested gzipped path explicitly or did not want gzip
+      if path =~ /\.gz\z/i || !want_gzip?(env)
+        res[1] = fheader(env, path, st)
+        res[2] = KF.open(path)
+      else
+        case gzbody = KF.tryopen("#{path}.gz")
+        when KF
+          gz_st = gzbody.stat
+          if gz_st.file? && gz_st.mtime == st.mtime
+            # yay! serve the gzipped version as the regular one
+            # this should be the most likely code path
+            res[1] = fheader(env, path, st, gz_st)
+            res[2] = gzbody
+          else
+            gzbody.close
+            res[1] = fheader(env, path, st)
+            res[2] = KF.open(path)
+          end
+        when :ENOENT, :EACCES
+          res[1] = fheader(env, path, st)
+          res[2] = KF.open(path)
+        else
+          res = r(500, gzbody.to_s, env)
+        end
+      end
+    end
+    res
+  rescue Errno::ENOENT # could get here from a race
+    r(404)
+  rescue Errno::EACCES # could get here from a race
+    r(403)
+  rescue => e
+    r(500, e.message, env)
+  end
+
+  def r(code, msg = nil, env = nil)
+    if env && logger = env["rack.logger"]
+      logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \
+                  "#{code} #{msg.inspect}")
+    end
+
+    if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code)
+      [ code, {}, [] ]
+    else
+      h = { 'Content-Type' => 'text/plain', 'Content-Length' => '0' }
+      [ code, h, [] ]
+    end
+  end
+end