pitchfork 0.1.0

data/lib/pitchfork/tee_input.rb

@@ -0,0 +1,133 @@
+# -*- encoding: binary -*-
+
+module Pitchfork
+  # Acts like tee(1) on an input input to provide a input-like stream
+  # while providing rewindable semantics through a File/StringIO backing
+  # store.  On the first pass, the input is only read on demand so your
+  # Rack application can use input notification (upload progress and
+  # like).  This should fully conform to the Rack::Lint::InputWrapper
+  # specification on the public API.  This class is intended to be a
+  # strict interpretation of Rack::Lint::InputWrapper functionality and
+  # will not support any deviations from it.
+  #
+  # When processing uploads, pitchfork exposes a TeeInput object under
+  # "rack.input" of the Rack environment by default.
+  class TeeInput < StreamInput
+    # The maximum size (in +bytes+) to buffer in memory before
+    # resorting to a temporary file.  Default is 112 kilobytes.
+    @@client_body_buffer_size = Pitchfork::Const::MAX_BODY # :nodoc:
+
+    # sets the maximum size of request bodies to buffer in memory,
+    # amounts larger than this are buffered to the filesystem
+    def self.client_body_buffer_size=(bytes) # :nodoc:
+      @@client_body_buffer_size = bytes
+    end
+
+    # returns the maximum size of request bodies to buffer in memory,
+    # amounts larger than this are buffered to the filesystem
+    def self.client_body_buffer_size # :nodoc:
+      @@client_body_buffer_size
+    end
+
+    # for Rack::TempfileReaper in rack 1.6+
+    def new_tmpio # :nodoc:
+      tmpio = Pitchfork::TmpIO.new
+      (@parser.env['rack.tempfiles'] ||= []) << tmpio
+      tmpio
+    end
+
+    # Initializes a new TeeInput object.  You normally do not have to call
+    # this unless you are writing an HTTP server.
+    def initialize(socket, request) # :nodoc:
+      @len = request.content_length
+      super
+      @tmp = @len && @len <= @@client_body_buffer_size ?
+             StringIO.new("") : new_tmpio
+    end
+
+    # :call-seq:
+    #   ios.size  => Integer
+    #
+    # Returns the size of the input.  For requests with a Content-Length
+    # header value, this will not read data off the socket and just return
+    # the value of the Content-Length header as an Integer.
+    #
+    # For Transfer-Encoding:chunked requests, this requires consuming
+    # all of the input stream before returning since there's no other
+    # way to determine the size of the request body beforehand.
+    #
+    # This method is no longer part of the Rack specification as of
+    # Rack 1.2, so its use is not recommended.  This method only exists
+    # for compatibility with Rack applications designed for Rack 1.1 and
+    # earlier.  Most applications should only need to call +read+ with a
+    # specified +length+ in a loop until it returns +nil+.
+    def size
+      @len and return @len
+      pos = @tmp.pos
+      consume!
+      @tmp.pos = pos
+      @len = @tmp.size
+    end
+
+    # :call-seq:
+    #   ios.read([length [, buffer ]]) => string, buffer, or nil
+    #
+    # Reads at most length bytes from the I/O stream, or to the end of
+    # file if length is omitted or is nil. length must be a non-negative
+    # integer or nil. If the optional buffer argument is present, it
+    # must reference a String, which will receive the data.
+    #
+    # At end of file, it returns nil or "" depend on length.
+    # ios.read() and ios.read(nil) returns "".
+    # ios.read(length [, buffer]) returns nil.
+    #
+    # If the Content-Length of the HTTP request is known (as is the common
+    # case for POST requests), then ios.read(length [, buffer]) will block
+    # until the specified length is read (or it is the last chunk).
+    # Otherwise, for uncommon "Transfer-Encoding: chunked" requests,
+    # ios.read(length [, buffer]) will return immediately if there is
+    # any data and only block when nothing is available (providing
+    # IO#readpartial semantics).
+    def read(*args)
+      @socket ? tee(super) : @tmp.read(*args)
+    end
+
+    # :call-seq:
+    #   ios.gets   => string or nil
+    #
+    # Reads the next ``line'' from the I/O stream; lines are separated
+    # by the global record separator ($/, typically "\n"). A global
+    # record separator of nil reads the entire unread contents of ios.
+    # Returns nil if called at the end of file.
+    # This takes zero arguments for strict Rack::Lint compatibility,
+    # unlike IO#gets.
+    def gets
+      @socket ? tee(super) : @tmp.gets
+    end
+
+    # :call-seq:
+    #   ios.rewind    => 0
+    #
+    # Positions the *ios* pointer to the beginning of input, returns
+    # the offset (zero) of the +ios+ pointer.  Subsequent reads will
+    # start from the beginning of the previously-buffered input.
+    def rewind
+      return 0 if 0 == @tmp.size
+      consume! if @socket
+      @tmp.rewind # Rack does not specify what the return value is here
+    end
+
+  private
+
+    # consumes the stream of the socket
+    def consume!
+      junk = ""
+      nil while read(@@io_chunk_size, junk)
+    end
+
+    def tee(buffer)
+      @tmp.write(buffer) if buffer
+      buffer
+    end
+  end
+end

data/lib/pitchfork/tmpio.rb

@@ -0,0 +1,35 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+require 'tmpdir'
+
+module Pitchfork
+  # some versions of Ruby had a broken Tempfile which didn't work
+  # well with unlinked files.  This one is much shorter, easier
+  # to understand, and slightly faster.
+  class TmpIO < File
+
+    # creates and returns a new File object.  The File is unlinked
+    # immediately, switched to binary mode, and userspace output
+    # buffering is disabled
+    def self.new
+      path = nil
+
+      # workaround File#path being tainted:
+      # https://bugs.ruby-lang.org/issues/14485
+      fp = begin
+        path = "#{Dir::tmpdir}/#{rand}"
+        super(path, RDWR|CREAT|EXCL, 0600)
+      rescue Errno::EEXIST
+        retry
+      end
+
+      unlink(path)
+      fp.binmode
+      fp.sync = true
+      fp
+    end
+
+    # pretend we're Tempfile for Rack::TempfileReaper
+    alias close! close
+  end
+end

data/lib/pitchfork/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Pitchfork
+  VERSION = "0.1.0"
+  module Const
+    UNICORN_VERSION = '6.1.0'
+  end
+end

data/lib/pitchfork/worker.rb

@@ -0,0 +1,244 @@
+# -*- encoding: binary -*-
+require "raindrops"
+
+module Pitchfork
+  # This class and its members can be considered a stable interface
+  # and will not change in a backwards-incompatible fashion between
+  # releases of pitchfork.  Knowledge of this class is generally not
+  # not needed for most users of pitchfork.
+  #
+  # Some users may want to access it in the before_fork/after_fork hooks.
+  # See the Pitchfork::Configurator RDoc for examples.
+  class Worker
+    # :stopdoc:
+    EXIT_SIGNALS = [:QUIT, :TERM]
+    @generation = 0
+    attr_accessor :nr, :pid, :generation
+    attr_reader :master
+
+    def initialize(nr, pid: nil, generation: 0)
+      @nr = nr
+      @pid = pid
+      @generation = generation
+      @mold = false
+      @to_io = @master = nil
+      @exiting = false
+      if nr
+        build_raindrops(nr)
+      else
+        promoted!
+      end
+    end
+
+    def meminfo
+      @meminfo ||= MemInfo.new(pid) if pid
+    end
+
+    def refresh
+      meminfo&.update
+    end
+
+    def exiting?
+      @exiting
+    end
+
+    def update(message)
+      message.class.members.each do |member|
+        send("#{member}=", message.public_send(member))
+      end
+
+      case message
+      when Message::WorkerPromoted, Message::PromoteWorker
+        promoted!
+      end
+    end
+
+    def register_to_master(control_socket)
+      create_socketpair!
+      message = Message::WorkerSpawned.new(@nr, Process.pid, generation, @master)
+      control_socket.sendmsg(message)
+      @master.close
+    end
+
+    def acknowlege_promotion(control_socket)
+      message = Message::WorkerPromoted.new(@nr, Process.pid, generation)
+      control_socket.sendmsg(message)
+    end
+
+    def promote(generation)
+      send_message_nonblock(Message::PromoteWorker.new(generation))
+    end
+
+    def spawn_worker(new_worker)
+      send_message_nonblock(Message::SpawnWorker.new(new_worker.nr))
+    end
+
+    def promoted!
+      @mold = true
+      @nr = nil
+      @drop_offset = 0
+      @tick_drop = MOLD_DROP
+    end
+
+    def mold?
+      @mold
+    end
+
+    def to_io # IO.select-compatible
+      @to_io.to_io
+    end
+
+    # master fakes SIGQUIT using this
+    def quit # :nodoc:
+      @master = @master.close if @master
+    end
+
+    def master=(socket)
+      @master = MessageSocket.new(socket)
+    end
+
+    # call a signal handler immediately without triggering EINTR
+    # We do not use the more obvious Process.kill(sig, $$) here since
+    # that signal delivery may be deferred.  We want to avoid signal delivery
+    # while the Rack app.call is running because some database drivers
+    # (e.g. ruby-pg) may cancel pending requests.
+    def fake_sig(sig) # :nodoc:
+      old_cb = trap(sig, "IGNORE")
+      old_cb.call
+    ensure
+      trap(sig, old_cb)
+    end
+
+    # master sends fake signals to children
+    def soft_kill(sig) # :nodoc:
+      signum = Signal.list[sig.to_s] or raise ArgumentError, "BUG: bad signal: #{sig.inspect}"
+
+      # Do not care in the odd case the buffer is full, here.
+      success = send_message_nonblock(Message::SoftKill.new(signum))
+      if success && EXIT_SIGNALS.include?(sig)
+        @exiting = true
+      end
+      success
+    end
+
+    # this only runs when the Rack app.call is not running
+    # act like a listener
+    def accept_nonblock(exception: nil) # :nodoc:
+      loop do
+        case buf = @to_io.recvmsg_nonblock(exception: false)
+        when :wait_readable # keep waiting
+          return false
+        when nil # EOF master died, but we are at a safe place to exit
+          fake_sig(:QUIT)
+          return false
+        when Message::SoftKill
+          # trigger the signal handler
+          fake_sig(buf.signum)
+          # keep looping, more signals may be queued
+        when Message
+          return buf
+        else
+          raise TypeError, "Unexpected recvmsg_nonblock returns: #{buf.inspect}"
+        end
+      end # loop, as multiple signals may be sent
+    rescue Errno::ECONNRESET
+      nil
+    end
+
+    # worker objects may be compared to just plain Integers
+    def ==(other) # :nodoc:
+      super || (!@nr.nil? && @nr == other)
+    end
+
+    # called in the worker process
+    def tick=(value) # :nodoc:
+      if mold?
+        MOLD_DROP[0] = value
+      else
+        @tick_drop[@drop_offset] = value
+      end
+    end
+
+    # called in the master process
+    def tick # :nodoc:
+      if mold?
+        MOLD_DROP[0]
+      else
+        @tick_drop[@drop_offset]
+      end
+    end
+
+    def reset
+      @requests_drop[@drop_offset] = 0
+    end
+
+    def requests_count
+      @requests_drop[@drop_offset]
+    end
+
+    def increment_requests_count
+      @requests_drop.incr(@drop_offset)
+    end
+
+    # called in both the master (reaping worker) and worker (SIGQUIT handler)
+    def close # :nodoc:
+      @master.close if @master
+      @to_io.close if @to_io
+    end
+
+    def create_socketpair!
+      @to_io, @master = Pitchfork.socketpair
+    end
+
+    def after_fork_in_child
+      @master.close
+    end
+
+    private
+
+    def pipe=(socket)
+      @master = MessageSocket.new(socket)
+    end
+
+    def send_message_nonblock(message)
+      success = false
+      begin
+        case @master.sendmsg_nonblock(message, exception: false)
+        when :wait_writable
+        else
+          success = true
+        end
+      rescue Errno::EPIPE
+        # worker will be reaped soon
+      end
+      success
+    end
+
+    MOLD_DROP = Raindrops.new(1)
+    PER_DROP = Raindrops::PAGE_SIZE / Raindrops::SIZE
+    TICK_DROPS = []
+    REQUEST_DROPS = []
+
+    class << self
+      # Since workers are created from another process, we have to
+      # pre-allocate the drops so they are shared between everyone.
+      #
+      # However this doesn't account for TTIN signals that increase the
+      # number of workers, but we should probably remove that feature too.
+      def preallocate_drops(workers_count)
+        0.upto(workers_count / PER_DROP) do |i|
+          TICK_DROPS[i] = Raindrops.new(PER_DROP)
+          REQUEST_DROPS[i] = Raindrops.new(PER_DROP)
+        end
+      end
+    end
+
+    def build_raindrops(drop_nr)
+      drop_index = drop_nr / PER_DROP
+      @drop_offset = drop_nr % PER_DROP
+      @tick_drop = TICK_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
+      @requests_drop = REQUEST_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
+      @tick_drop[@drop_offset] = @requests_drop[@drop_offset] = 0
+    end
+  end
+end

data/lib/pitchfork.rb

@@ -0,0 +1,158 @@
+# -*- encoding: binary -*-
+require 'etc'
+require 'stringio'
+require 'raindrops'
+require 'io/wait'
+
+begin
+  require 'rack'
+rescue LoadError
+  warn 'rack not available, functionality reduced'
+end
+
+# :stopdoc:
+# Pitchfork module containing all of the classes (include C extensions) for
+# running a Pitchfork web server.  It contains a minimalist HTTP server with just
+# enough functionality to service web application requests fast as possible.
+# :startdoc:
+
+# pitchfork exposes very little of an user-visible API and most of its
+# internals are subject to change.  pitchfork is designed to host Rack
+# applications, so applications should be written against the Rack SPEC
+# and not pitchfork internals.
+module Pitchfork
+
+  # Raised inside TeeInput when a client closes the socket inside the
+  # application dispatch.  This is always raised with an empty backtrace
+  # since there is nothing in the application stack that is responsible
+  # for client shutdowns/disconnects.  This exception is visible to Rack
+  # applications unless PrereadInput middleware is loaded.  This
+  # is a subclass of the standard EOFError class and applications should
+  # not rescue it explicitly, but rescue EOFError instead.
+  ClientShutdown = Class.new(EOFError)
+
+  BootFailure = Class.new(StandardError)
+
+  # :stopdoc:
+
+  # This returns a lambda to pass in as the app, this does not "build" the
+  # app The returned lambda will be called when it is
+  # time to build the app.
+  def self.builder(ru, op)
+    # allow Configurator to parse cli switches embedded in the ru file
+    op = Pitchfork::Configurator::RACKUP.merge!(:file => ru, :optparse => op)
+    if ru =~ /\.ru$/ && !defined?(Rack::Builder)
+      abort "rack and Rack::Builder must be available for processing #{ru}"
+    end
+
+    # always called after config file parsing, may be called after forking
+    lambda do |_, server|
+      inner_app = case ru
+      when /\.ru$/
+        raw = File.read(ru)
+        raw.sub!(/^__END__\n.*/, '')
+        eval("Rack::Builder.new {(\n#{raw}\n)}.to_app", TOPLEVEL_BINDING, ru)
+      else
+        require ru
+        Object.const_get(File.basename(ru, '.rb').capitalize)
+      end
+
+      if $DEBUG
+        require 'pp'
+        pp({ :inner_app => inner_app })
+      end
+
+      return inner_app unless server.default_middleware
+
+      middleware = { # order matters
+        ContentLength: nil,
+        Chunked: nil,
+        CommonLogger: [ $stderr ],
+        ShowExceptions: nil,
+        Lint: nil,
+        TempfileReaper: nil,
+      }
+
+      # return value, matches rackup defaults based on env
+      # Pitchfork does not support persistent connections, but Rainbows!
+      # and Zbatery both do.  Users accustomed to the Rack::Server default
+      # middlewares will need ContentLength/Chunked middlewares.
+      case ENV["RACK_ENV"]
+      when "development"
+      when "deployment"
+        middleware.delete(:ShowExceptions)
+        middleware.delete(:Lint)
+      else
+        return inner_app
+      end
+      Rack::Builder.new do
+        middleware.each do |m, args|
+          use(Rack.const_get(m), *args) if Rack.const_defined?(m)
+        end
+        run inner_app
+      end.to_app
+    end
+  end
+
+  # returns an array of strings representing TCP listen socket addresses
+  # and Unix domain socket paths.  This is useful for use with
+  # Raindrops::Middleware under Linux: https://yhbt.net/raindrops/
+  def self.listener_names
+    Pitchfork::HttpServer::LISTENERS.map do |io|
+      Pitchfork::SocketHelper.sock_name(io)
+    end
+  end
+
+  def self.log_error(logger, prefix, exc)
+    message = exc.message
+    message = message.dump if /[[:cntrl:]]/ =~ message
+    logger.error "#{prefix}: #{message} (#{exc.class})"
+    exc.backtrace.each { |line| logger.error(line) }
+  end
+
+  F_SETPIPE_SZ = 1031 if RUBY_PLATFORM =~ /linux/
+
+  def self.pipe # :nodoc:
+    IO.pipe.each do |io|
+      # shrink pipes to minimize impact on /proc/sys/fs/pipe-user-pages-soft
+      # limits.
+      if defined?(F_SETPIPE_SZ)
+        begin
+          io.fcntl(F_SETPIPE_SZ, Raindrops::PAGE_SIZE)
+        rescue Errno::EINVAL
+          # old kernel
+        rescue Errno::EPERM
+          # resizes fail if Linux is close to the pipe limit for the user
+          # or if the user does not have permissions to resize
+        end
+      end
+    end
+  end
+
+  @socket_type = :SOCK_SEQPACKET
+  def self.socketpair
+    pair = UNIXSocket.socketpair(@socket_type).map { |s| MessageSocket.new(s) }
+    pair[0].close_write
+    pair[1].close_read
+    pair
+  rescue Errno::EPROTONOSUPPORT
+    if @socket_type == :SOCK_SEQPACKET
+      # macOS and very old linuxes don't support SOCK_SEQPACKET (SCTP).
+      # In such case we can fallback to SOCK_STREAM (TCP)
+      warn("SEQPACKET (SCTP) isn't supported, falling back to STREAM")
+      @socket_type = :SOCK_STREAM
+      retry
+    else
+      raise
+    end
+  end
+  # :startdoc:
+end
+# :enddoc:
+
+%w(
+  const socket_helper stream_input tee_input mem_info children message http_parser
+  refork_condition mold_selector configurator tmpio http_response worker http_server
+).each do |s|
+  require_relative "pitchfork/#{s}"
+end

data/pitchfork.gemspec

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "lib/pitchfork/version"
+
+Gem::Specification.new do |s|
+  s.name = %q{pitchfork}
+  s.version = Pitchfork::VERSION
+  s.authors = ['Jean Boussier']
+  s.email = ["jean.boussier@gmail.com"]
+  s.summary = 'Rack HTTP server for fast clients and Unix'
+  s.description = File.read('README.md').split("\n\n")[1]
+  s.extensions = %w(ext/pitchfork_http/extconf.rb)
+  s.files = Dir.chdir(File.expand_path('..', __FILE__)) do
+    %x(git ls-files -z).split("\x0").reject { |f| f.match(%r{^(test|spec|features|bin|\.)/}) }
+  end
+  s.bindir = "exe"
+  s.executables = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.homepage = 'https://github.com/Shopify/pitchfork'
+
+  s.required_ruby_version = ">= 2.5.0"
+
+  s.add_dependency(%q<raindrops>, '~> 0.7')
+  s.add_dependency(%q<rack>, '>= 2.0')
+
+  # Note: To avoid ambiguity, we intentionally avoid the SPDX-compatible
+  # 'Ruby' here since Ruby 1.9.3 switched to BSD-2-Clause, but we
+  # inherited our license from Mongrel when Ruby was at 1.8.
+  # We cannot automatically switch licenses when Ruby changes.
+  s.licenses = ['GPL-2.0+', 'Ruby']
+end