unicorn-shopify 4.8.2.5.23

data/lib/unicorn/launcher.rb

@@ -0,0 +1,62 @@
+# -*- encoding: binary -*-
+
+# :enddoc:
+$stdout.sync = $stderr.sync = true
+$stdin.binmode
+$stdout.binmode
+$stderr.binmode
+
+require 'unicorn'
+
+module Unicorn::Launcher
+
+  # We don't do a lot of standard daemonization stuff:
+  #   * umask is whatever was set by the parent process at startup
+  #     and can be set in config.ru and config_file, so making it
+  #     0000 and potentially exposing sensitive log data can be bad
+  #     policy.
+  #   * don't bother to chdir("/") here since unicorn is designed to
+  #     run inside APP_ROOT.  Unicorn will also re-chdir() to
+  #     the directory it was started in when being re-executed
+  #     to pickup code changes if the original deployment directory
+  #     is a symlink or otherwise got replaced.
+  def self.daemonize!(options)
+    cfg = Unicorn::Configurator
+    $stdin.reopen("/dev/null")
+
+    # We only start a new process group if we're not being reexecuted
+    # and inheriting file descriptors from our parent
+    unless ENV['UNICORN_FD']
+      # grandparent - reads pipe, exits when master is ready
+      #  \_ parent  - exits immediately ASAP
+      #      \_ unicorn master - writes to pipe when ready
+
+      rd, wr = IO.pipe
+      grandparent = $$
+      if fork
+        wr.close # grandparent does not write
+      else
+        rd.close # unicorn master does not read
+        Process.setsid
+        exit if fork # parent dies now
+      end
+
+      if grandparent == $$
+        # this will block until HttpServer#join runs (or it dies)
+        master_pid = (rd.readpartial(16) rescue nil).to_i
+        unless master_pid > 1
+          warn "master failed to start, check stderr log for details"
+          exit!(1)
+        end
+        exit 0
+      else # unicorn master process
+        options[:ready_pipe] = wr
+      end
+    end
+    # $stderr/$stderr can/will be redirected separately in the Unicorn config
+    cfg::DEFAULTS[:stderr_path] ||= "/dev/null"
+    cfg::DEFAULTS[:stdout_path] ||= "/dev/null"
+    cfg::RACKUP[:daemonized] = true
+  end
+
+end

data/lib/unicorn/oob_gc.rb

@@ -0,0 +1,81 @@
+# -*- encoding: binary -*-
+
+# Strongly consider https://github.com/tmm1/gctools if using Ruby 2.1+
+# It is built on new APIs in Ruby 2.1, so it is more intelligent than
+# this historical implementation.
+#
+# Users on Ruby 2.0 (not 2.1+) may also want to check out
+# lib/middleware/unicorn_oobgc.rb from the Discourse project
+# (https://github.com/discourse/discourse)
+#
+# The following information is only for historical versions of Ruby.
+#
+# Runs GC after requests, after closing the client socket and
+# before attempting to accept more connections.
+#
+# This shouldn't hurt overall performance as long as the server cluster
+# is at <50% CPU capacity, and improves the performance of most memory
+# intensive requests.  This serves to improve _client-visible_
+# performance (possibly at the cost of overall performance).
+#
+# Increasing the number of +worker_processes+ may be necessary to
+# improve average client response times because some of your workers
+# will be busy doing GC and unable to service clients.  Think of
+# using more workers with this module as a poor man's concurrent GC.
+#
+# We'll call GC after each request is been written out to the socket, so
+# the client never sees the extra GC hit it.
+#
+# This middleware is _only_ effective for applications that use a lot
+# of memory, and will hurt simpler apps/endpoints that can process
+# multiple requests before incurring GC.
+#
+# This middleware is only designed to work with unicorn, as it harms
+# performance with keepalive-enabled servers.
+#
+# Example (in config.ru):
+#
+#     require 'unicorn/oob_gc'
+#
+#     # GC ever two requests that hit /expensive/foo or /more_expensive/foo
+#     # in your app.  By default, this will GC once every 5 requests
+#     # for all endpoints in your app
+#     use Unicorn::OobGC, 2, %r{\A/(?:expensive/foo|more_expensive/foo)}
+#
+# Feedback from users of early implementations of this module:
+# * http://comments.gmane.org/gmane.comp.lang.ruby.unicorn.general/486
+# * http://article.gmane.org/gmane.comp.lang.ruby.unicorn.general/596
+module Unicorn::OobGC
+
+  # this pretends to be Rack middleware because it used to be
+  # But we need to hook into unicorn internals so we need to close
+  # the socket before clearing the request env.
+  #
+  # +interval+ is the number of requests matching the +path+ regular
+  # expression before invoking GC.
+  def self.new(app, interval = 5, path = %r{\A/})
+    @@nr = interval
+    self.const_set :OOBGC_PATH, path
+    self.const_set :OOBGC_INTERVAL, interval
+    ObjectSpace.each_object(Unicorn::HttpServer) do |s|
+      s.extend(self)
+      self.const_set :OOBGC_ENV, s.instance_variable_get(:@request).env
+    end
+    app # pretend to be Rack middleware since it was in the past
+  end
+
+  #:stopdoc:
+  PATH_INFO = "PATH_INFO"
+  def process_client(client)
+    super(client) # Unicorn::HttpServer#process_client
+    if OOBGC_PATH =~ OOBGC_ENV[PATH_INFO] && ((@@nr -= 1) <= 0)
+      @@nr = OOBGC_INTERVAL
+      OOBGC_ENV.clear
+      disabled = GC.enable
+      GC.start
+      GC.disable if disabled
+    end
+  end
+
+  # :startdoc:
+end

data/lib/unicorn/preread_input.rb

@@ -0,0 +1,33 @@
+# -*- encoding: binary -*-
+
+module Unicorn
+# This middleware is used to ensure input is buffered to memory
+# or disk (depending on size) before the application is dispatched
+# by entirely consuming it (from TeeInput) beforehand.
+#
+# Usage (in config.ru):
+#
+#     require 'unicorn/preread_input'
+#     if defined?(Unicorn)
+#       use Unicorn::PrereadInput
+#     end
+#     run YourApp.new
+class PrereadInput
+
+  # :stopdoc:
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    buf = ""
+    input = env["rack.input"]
+    if input.respond_to?(:rewind)
+      true while input.read(16384, buf)
+      input.rewind
+    end
+    @app.call(env)
+  end
+  # :startdoc:
+end
+end

data/lib/unicorn/socket_helper.rb

@@ -0,0 +1,197 @@
+# -*- encoding: binary -*-
+# :enddoc:
+require 'socket'
+
+module Unicorn
+  module SocketHelper
+
+    # internal interface, only used by Rainbows!/Zbatery
+    DEFAULTS = {
+      # The semantics for TCP_DEFER_ACCEPT changed in Linux 2.6.32+
+      # with commit d1b99ba41d6c5aa1ed2fc634323449dd656899e9
+      # This change shouldn't affect Unicorn users behind nginx (a
+      # value of 1 remains an optimization), but Rainbows! users may
+      # want to use a higher value on Linux 2.6.32+ to protect against
+      # denial-of-service attacks
+      :tcp_defer_accept => 1,
+
+      # FreeBSD, we need to override this to 'dataready' if we
+      # eventually support non-HTTP/1.x
+      :accept_filter => 'httpready',
+
+      # same default value as Mongrel
+      :backlog => 1024,
+
+      # favor latency over bandwidth savings
+      :tcp_nopush => nil,
+      :tcp_nodelay => true,
+    }
+
+    # configure platform-specific options (only tested on Linux 2.6 so far)
+    def accf_arg(af_name)
+      [ af_name, nil ].pack('a16a240')
+    end if RUBY_PLATFORM =~ /freebsd/ && Socket.const_defined?(:SO_ACCEPTFILTER)
+
+    def set_tcp_sockopt(sock, opt)
+      # just in case, even LANs can break sometimes.  Linux sysadmins
+      # can lower net.ipv4.tcp_keepalive_* sysctl knobs to very low values.
+      Socket.const_defined?(:SO_KEEPALIVE) and
+        sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1)
+
+      if Socket.const_defined?(:TCP_NODELAY)
+        val = opt[:tcp_nodelay]
+        val = DEFAULTS[:tcp_nodelay] if val.nil?
+        sock.setsockopt(:IPPROTO_TCP, :TCP_NODELAY, val ? 1 : 0)
+      end
+
+      val = opt[:tcp_nopush]
+      unless val.nil?
+        if Socket.const_defined?(:TCP_CORK) # Linux
+          sock.setsockopt(:IPPROTO_TCP, :TCP_CORK, val)
+        elsif Socket.const_defined?(:TCP_NOPUSH) # FreeBSD
+          sock.setsockopt(:IPPROTO_TCP, :TCP_NOPUSH, val)
+        end
+      end
+
+      # No good reason to ever have deferred accepts off in single-threaded
+      # servers (except maybe benchmarking)
+      if Socket.const_defined?(:TCP_DEFER_ACCEPT)
+        # this differs from nginx, since nginx doesn't allow us to
+        # configure the the timeout...
+        seconds = opt[:tcp_defer_accept]
+        seconds = DEFAULTS[:tcp_defer_accept] if [true,nil].include?(seconds)
+        seconds = 0 unless seconds # nil/false means disable this
+        sock.setsockopt(:IPPROTO_TCP, :TCP_DEFER_ACCEPT, seconds)
+      elsif respond_to?(:accf_arg)
+        name = opt[:accept_filter]
+        name = DEFAULTS[:accept_filter] if name.nil?
+        begin
+          sock.setsockopt(:SOL_SOCKET, :SO_ACCEPTFILTER, accf_arg(name))
+        rescue => e
+          logger.error("#{sock_name(sock)} " \
+                       "failed to set accept_filter=#{name} (#{e.inspect})")
+        end
+      end
+    end
+
+    def set_server_sockopt(sock, opt)
+      opt = DEFAULTS.merge(opt || {})
+
+      TCPSocket === sock and set_tcp_sockopt(sock, opt)
+
+      rcvbuf, sndbuf = opt.values_at(:rcvbuf, :sndbuf)
+      if rcvbuf || sndbuf
+        log_buffer_sizes(sock, "before: ")
+        sock.setsockopt(:SOL_SOCKET, :SO_RCVBUF, rcvbuf) if rcvbuf
+        sock.setsockopt(:SOL_SOCKET, :SO_SNDBUF, sndbuf) if sndbuf
+        log_buffer_sizes(sock, " after: ")
+      end
+      sock.listen(opt[:backlog])
+      rescue => e
+        Unicorn.log_error(logger, "#{sock_name(sock)} #{opt.inspect}", e)
+    end
+
+    def log_buffer_sizes(sock, pfx = '')
+      rcvbuf = sock.getsockopt(:SOL_SOCKET, :SO_RCVBUF).int
+      sndbuf = sock.getsockopt(:SOL_SOCKET, :SO_SNDBUF).int
+      logger.info "#{pfx}#{sock_name(sock)} rcvbuf=#{rcvbuf} sndbuf=#{sndbuf}"
+    end
+
+    # creates a new server, socket. address may be a HOST:PORT or
+    # an absolute path to a UNIX socket.  address can even be a Socket
+    # object in which case it is immediately returned
+    def bind_listen(address = '0.0.0.0:8080', opt = {})
+      return address unless String === address
+
+      sock = if address[0] == ?/
+        if File.exist?(address)
+          if File.socket?(address)
+            begin
+              UNIXSocket.new(address).close
+              # fall through, try to bind(2) and fail with EADDRINUSE
+              # (or succeed from a small race condition we can't sanely avoid).
+            rescue Errno::ECONNREFUSED
+              logger.info "unlinking existing socket=#{address}"
+              File.unlink(address)
+            end
+          else
+            raise ArgumentError,
+                  "socket=#{address} specified but it is not a socket!"
+          end
+        end
+        old_umask = File.umask(opt[:umask] || 0)
+        begin
+          Kgio::UNIXServer.new(address)
+        ensure
+          File.umask(old_umask)
+        end
+      elsif /\A\[([a-fA-F0-9:]+)\]:(\d+)\z/ =~ address
+        new_tcp_server($1, $2.to_i, opt.merge(:ipv6=>true))
+      elsif /\A(\d+\.\d+\.\d+\.\d+):(\d+)\z/ =~ address
+        new_tcp_server($1, $2.to_i, opt)
+      else
+        raise ArgumentError, "Don't know how to bind: #{address}"
+      end
+      set_server_sockopt(sock, opt)
+      sock
+    end
+
+    def new_tcp_server(addr, port, opt)
+      # n.b. we set FD_CLOEXEC in the workers
+      sock = Socket.new(opt[:ipv6] ? :AF_INET6 : :AF_INET, :SOCK_STREAM)
+      if opt.key?(:ipv6only)
+        Socket.const_defined?(:IPV6_V6ONLY) or
+          abort "Socket::IPV6_V6ONLY not defined, upgrade Ruby and/or your OS"
+        sock.setsockopt(:IPPROTO_IPV6, :IPV6_V6ONLY, opt[:ipv6only] ? 1 : 0)
+      end
+      sock.setsockopt(:SOL_SOCKET, :SO_REUSEADDR, 1)
+      if Socket.const_defined?(:SO_REUSEPORT) && opt[:reuseport]
+        sock.setsockopt(:SOL_SOCKET, :SO_REUSEPORT, 1)
+      end
+      sock.bind(Socket.pack_sockaddr_in(port, addr))
+      sock.autoclose = false
+      Kgio::TCPServer.for_fd(sock.fileno)
+    end
+
+    # returns rfc2732-style (e.g. "[::1]:666") addresses for IPv6
+    def tcp_name(sock)
+      port, addr = Socket.unpack_sockaddr_in(sock.getsockname)
+      addr.include?(':') ? "[#{addr}]:#{port}" : "#{addr}:#{port}"
+    end
+    module_function :tcp_name
+
+    # Returns the configuration name of a socket as a string.  sock may
+    # be a string value, in which case it is returned as-is
+    # Warning: TCP sockets may not always return the name given to it.
+    def sock_name(sock)
+      case sock
+      when String then sock
+      when UNIXServer
+        Socket.unpack_sockaddr_un(sock.getsockname)
+      when TCPServer
+        tcp_name(sock)
+      when Socket
+        begin
+          tcp_name(sock)
+        rescue ArgumentError
+          Socket.unpack_sockaddr_un(sock.getsockname)
+        end
+      else
+        raise ArgumentError, "Unhandled class #{sock.class}: #{sock.inspect}"
+      end
+    end
+
+    module_function :sock_name
+
+    # casts a given Socket to be a TCPServer or UNIXServer
+    def server_cast(sock)
+      begin
+        Socket.unpack_sockaddr_in(sock.getsockname)
+        Kgio::TCPServer.for_fd(sock.fileno)
+      rescue ArgumentError
+        Kgio::UNIXServer.for_fd(sock.fileno)
+      end
+    end
+
+  end # module SocketHelper
+end # module Unicorn

data/lib/unicorn/stream_input.rb

@@ -0,0 +1,146 @@
+# -*- encoding: binary -*-
+
+# When processing uploads, Unicorn may expose a StreamInput object under
+# "rack.input" of the (future) Rack (2.x) environment.
+class Unicorn::StreamInput
+  # The I/O chunk size (in +bytes+) for I/O operations where
+  # the size cannot be user-specified when a method is called.
+  # The default is 16 kilobytes.
+  @@io_chunk_size = Unicorn::Const::CHUNK_SIZE
+
+  # Initializes a new StreamInput object.  You normally do not have to call
+  # this unless you are writing an HTTP server.
+  def initialize(socket, request)
+    @chunked = request.content_length.nil?
+    @socket = socket
+    @parser = request
+    @buf = request.buf
+    @rbuf = ''
+    @bytes_read = 0
+    filter_body(@rbuf, @buf) unless @buf.empty?
+  end
+
+  # :call-seq:
+  #   ios.read([length [, buffer ]]) => string, buffer, or nil
+  #
+  # Reads at most length bytes from the I/O stream, or to the end of
+  # file if length is omitted or is nil. length must be a non-negative
+  # integer or nil. If the optional buffer argument is present, it
+  # must reference a String, which will receive the data.
+  #
+  # At end of file, it returns nil or '' depend on length.
+  # ios.read() and ios.read(nil) returns ''.
+  # ios.read(length [, buffer]) returns nil.
+  #
+  # If the Content-Length of the HTTP request is known (as is the common
+  # case for POST requests), then ios.read(length [, buffer]) will block
+  # until the specified length is read (or it is the last chunk).
+  # Otherwise, for uncommon "Transfer-Encoding: chunked" requests,
+  # ios.read(length [, buffer]) will return immediately if there is
+  # any data and only block when nothing is available (providing
+  # IO#readpartial semantics).
+  def read(length = nil, rv = '')
+    if length
+      if length <= @rbuf.size
+        length < 0 and raise ArgumentError, "negative length #{length} given"
+        rv.replace(@rbuf.slice!(0, length))
+      else
+        to_read = length - @rbuf.size
+        rv.replace(@rbuf.slice!(0, @rbuf.size))
+        until to_read == 0 || eof? || (rv.size > 0 && @chunked)
+          @socket.kgio_read(to_read, @buf) or eof!
+          filter_body(@rbuf, @buf)
+          rv << @rbuf
+          to_read -= @rbuf.size
+        end
+        @rbuf.replace('')
+      end
+      rv = nil if rv.empty? && length != 0
+    else
+      read_all(rv)
+    end
+    rv
+  end
+
+  # :call-seq:
+  #   ios.gets   => string or nil
+  #
+  # Reads the next ``line'' from the I/O stream; lines are separated
+  # by the global record separator ($/, typically "\n"). A global
+  # record separator of nil reads the entire unread contents of ios.
+  # Returns nil if called at the end of file.
+  # This takes zero arguments for strict Rack::Lint compatibility,
+  # unlike IO#gets.
+  def gets
+    sep = $/
+    if sep.nil?
+      read_all(rv = '')
+      return rv.empty? ? nil : rv
+    end
+    re = /\A(.*?#{Regexp.escape(sep)})/
+
+    begin
+      @rbuf.sub!(re, '') and return $1
+      return @rbuf.empty? ? nil : @rbuf.slice!(0, @rbuf.size) if eof?
+      @socket.kgio_read(@@io_chunk_size, @buf) or eof!
+      filter_body(once = '', @buf)
+      @rbuf << once
+    end while true
+  end
+
+  # :call-seq:
+  #   ios.each { |line| block }  => ios
+  #
+  # Executes the block for every ``line'' in *ios*, where lines are
+  # separated by the global record separator ($/, typically "\n").
+  def each
+    while line = gets
+      yield line
+    end
+
+    self # Rack does not specify what the return value is here
+  end
+
+private
+
+  def eof?
+    if @parser.body_eof?
+      while @chunked && ! @parser.parse
+        once = @socket.kgio_read(@@io_chunk_size) or eof!
+        @buf << once
+      end
+      @socket = nil
+      true
+    else
+      false
+    end
+  end
+
+  def filter_body(dst, src)
+    rv = @parser.filter_body(dst, src)
+    @bytes_read += dst.size
+    rv
+  end
+
+  def read_all(dst)
+    dst.replace(@rbuf)
+    @socket or return
+    until eof?
+      @socket.kgio_read(@@io_chunk_size, @buf) or eof!
+      filter_body(@rbuf, @buf)
+      dst << @rbuf
+    end
+    ensure
+      @rbuf.replace('')
+  end
+
+  def eof!
+    # in case client only did a premature shutdown(SHUT_WR)
+    # we do support clients that shutdown(SHUT_WR) after the
+    # _entire_ request has been sent, and those will not have
+    # raised EOFError on us.
+    @socket.shutdown if @socket
+  ensure
+    raise Unicorn::ClientShutdown, "bytes_read=#{@bytes_read}", []
+  end
+end