pitchfork 0.2.0 → 0.3.0

data/lib/pitchfork/children.rb

@@ -29,7 +29,6 @@ module Pitchfork
     def register_mold(mold)
       @pending_molds[mold.pid] = mold
       @children[mold.pid] = mold
-      @mold = mold
     end
 
     def fetch(pid)
@@ -37,21 +36,29 @@ module Pitchfork
     end
 
     def update(message)
-      child = @children[message.pid] || (message.nr && @workers[message.nr])
-      old_nr = child.nr
+      case message
+      when Message::MoldSpawned
+        mold = Worker.new(nil)
+        mold.update(message)
+        @pending_molds[mold.pid] = mold
+        @children[mold.pid] = mold
+        return mold
+      end
 
+      child = @children[message.pid] || (message.nr && @workers[message.nr])
       child.update(message)
 
       if child.mold?
-        @workers.delete(old_nr)
         @pending_molds.delete(child.pid)
         @molds[child.pid] = child
         @mold = child
       end
+
       if child.pid
         @children[child.pid] = child
         @pending_workers.delete(child.nr)
       end
+
       child
     end
 
@@ -73,7 +80,6 @@ module Pitchfork
     end
 
     def promote(worker)
-      @pending_molds[worker.pid] = worker
       worker.promote(self.last_generation += 1)
     end
 
@@ -81,6 +87,10 @@ module Pitchfork
       !(@pending_workers.empty? && @pending_molds.empty?)
     end
 
+    def restarting_workers_count
+      @pending_workers.size + @workers.count { |_, w| w.exiting? }
+    end
+
     def pending_promotion?
       !@pending_molds.empty?
     end

data/lib/pitchfork/chunked.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require "rack"
+if Rack::VERSION.first >= 3
+  require "rack/constants"
+  require "rack/utils"
+end
+
+module Pitchfork
+  # Middleware that applies chunked transfer encoding to response bodies
+  # when the response does not include a content-length header.
+  #
+  # This supports the trailer response header to allow the use of trailing
+  # headers in the chunked encoding.  However, using this requires you manually
+  # specify a response body that supports a +trailers+ method.  Example:
+  #
+  #   [200, { 'trailer' => 'expires'}, ["Hello", "World"]]
+  #   # error raised
+  #
+  #   body = ["Hello", "World"]
+  #   def body.trailers
+  #     { 'expires' => Time.now.to_s }
+  #   end
+  #   [200, { 'trailer' => 'expires'}, body]
+  #   # No exception raised
+  class Chunked
+    include Rack::Utils
+
+    STATUS_WITH_NO_ENTITY_BODY = Hash[((100..199).to_a << 204 << 304).product([true])]
+
+    # A body wrapper that emits chunked responses.
+    class Body
+      TERM = "\r\n"
+      TAIL = "0#{TERM}"
+
+      # Store the response body to be chunked.
+      def initialize(body)
+        @body = body
+      end
+
+      # For each element yielded by the response body, yield
+      # the element in chunked encoding.
+      def each(&block)
+        term = TERM
+        @body.each do |chunk|
+          size = chunk.bytesize
+          next if size == 0
+
+          yield [size.to_s(16), term, chunk.b, term].join
+        end
+        yield TAIL
+        yield_trailers(&block)
+        yield term
+      end
+
+      # Close the response body if the response body supports it.
+      def close
+        @body.close if @body.respond_to?(:close)
+      end
+
+      private
+
+      # Do nothing as this class does not support trailer headers.
+      def yield_trailers
+      end
+    end
+
+    # A body wrapper that emits chunked responses and also supports
+    # sending Trailer headers.  Note that the response body provided to
+    # initialize must have a +trailers+ method that returns a hash
+    # of trailer headers, and the rack response itself should have a
+    # Trailer header listing the headers that the +trailers+ method
+    # will return.
+    class TrailerBody < Body
+      private
+
+      # Yield strings for each trailer header.
+      def yield_trailers
+        @body.trailers.each_pair do |k, v|
+          yield "#{k}: #{v}\r\n"
+        end
+      end
+    end
+
+    def initialize(app)
+      @app = app
+    end
+
+    # Whether the HTTP version supports chunked encoding (HTTP 1.1 does).
+    def chunkable_version?(ver)
+      case ver
+      # pre-HTTP/1.0 (informally "HTTP/0.9") HTTP requests did not have
+      # a version (nor response headers)
+      when 'HTTP/1.0', nil, 'HTTP/0.9'
+        false
+      else
+        true
+      end
+    end
+
+    # If the rack app returns a response that should have a body,
+    # but does not have content-length or transfer-encoding headers,
+    # modify the response to use chunked transfer-encoding.
+    def call(env)
+      status, headers, body = response = @app.call(env)
+
+      if chunkable_version?(env[Rack::SERVER_PROTOCOL]) &&
+         !STATUS_WITH_NO_ENTITY_BODY.key?(status.to_i) &&
+         !headers[Rack::CONTENT_LENGTH] &&
+         !headers[Rack::TRANSFER_ENCODING]
+
+        headers[Rack::TRANSFER_ENCODING] = 'chunked'
+        if headers['trailer']
+          response[2] = TrailerBody.new(body)
+        else
+          response[2] = Body.new(body)
+        end
+      end
+
+      response
+    end
+  end
+end

data/lib/pitchfork/configurator.rb

@@ -30,11 +30,11 @@ module Pitchfork
       :timeout => 20,
       :logger => Logger.new($stderr),
       :worker_processes => 1,
-      :after_fork => lambda { |server, worker|
+      :after_worker_fork => lambda { |server, worker|
         server.logger.info("worker=#{worker.nr} gen=#{worker.generation} pid=#{$$} spawned")
       },
-      :after_promotion => lambda { |server, worker|
-        server.logger.info("gen=#{worker.generation} pid=#{$$} promoted")
+      :after_mold_fork => lambda { |server, worker|
+        server.logger.info("mold gen=#{worker.generation} pid=#{$$} spawned")
       },
       :after_worker_exit => lambda { |server, worker, status|
         m = if worker.nil?
@@ -119,12 +119,12 @@ module Pitchfork
       set[:logger] = obj
     end
 
-    def after_fork(*args, &block)
-      set_hook(:after_fork, block_given? ? block : args[0])
+    def after_worker_fork(*args, &block)
+      set_hook(:after_worker_fork, block_given? ? block : args[0])
     end
 
-    def after_promotion(*args, &block)
-      set_hook(:after_promotion, block_given? ? block : args[0])
+    def after_mold_fork(*args, &block)
+      set_hook(:after_mold_fork, block_given? ? block : args[0])
     end
 
     def after_worker_ready(*args, &block)

data/lib/pitchfork/http_parser.rb

@@ -1,7 +1,6 @@
 # -*- encoding: binary -*-
 # :enddoc:
 # no stable API here
-require 'pitchfork/pitchfork_http'
 
 module Pitchfork
   class HttpParser

data/lib/pitchfork/http_server.rb

@@ -10,7 +10,7 @@ module Pitchfork
   class HttpServer
     # :stopdoc:
     attr_accessor :app, :timeout, :worker_processes,
-                  :after_fork, :after_promotion,
+                  :after_worker_fork, :after_mold_fork,
                   :listener_opts, :children,
                   :orig_app, :config, :ready_pipe,
                   :default_middleware, :early_hints
@@ -27,8 +27,6 @@ module Pitchfork
 
     NOOP = '.'
 
-    REFORKING_AVAILABLE = Pitchfork::CHILD_SUBREAPER_AVAILABLE || Process.pid == 1
-
     # :startdoc:
     # This Hash is considered a stable interface and changing its contents
     # will allow you to switch between different installations of Pitchfork
@@ -63,7 +61,7 @@ module Pitchfork
       @exit_status = 0
       @app = app
       @respawn = false
-      @last_check = time_now
+      @last_check = Pitchfork.time_now
       @promotion_lock = Flock.new("pitchfork-promotion")
 
       options = options.dup
@@ -131,7 +129,7 @@ module Pitchfork
       else
         build_app!
         bind_listeners!
-        after_promotion.call(self, Worker.new(nil, pid: $$).promoted!)
+        after_mold_fork.call(self, Worker.new(nil, pid: $$).promoted!)
       end
 
       if sync
@@ -178,7 +176,7 @@ module Pitchfork
 
     # add a given address to the +listeners+ set, idempotently
     # Allows workers to add a private, per-process listener via the
-    # after_fork hook.  Very useful for debugging and testing.
+    # after_worker_fork hook.  Very useful for debugging and testing.
     # +:tries+ may be specified as an option for the number of times
     # to retry, and +:delay+ may be specified as the time in seconds
     # to delay between retries.
@@ -259,7 +257,7 @@ module Pitchfork
       when nil
         # avoid murdering workers after our master process (or the
         # machine) comes out of suspend/hibernation
-        if (@last_check + @timeout) >= (@last_check = time_now)
+        if (@last_check + @timeout) >= (@last_check = Pitchfork.time_now)
           sleep_time = murder_lazy_workers
         else
           sleep_time = @timeout/2.0 + 1
@@ -289,13 +287,15 @@ module Pitchfork
         worker = @children.update(message)
         # TODO: should we send a message to the worker to acknowledge?
         logger.info "worker=#{worker.nr} pid=#{worker.pid} registered"
-      when Message::WorkerPromoted
+      when Message::MoldSpawned
+        new_mold = @children.update(message)
+        logger.info("mold pid=#{new_mold.pid} gen=#{new_mold.generation} spawned")
+      when Message::MoldReady
         old_molds = @children.molds
-        new_mold = @children.fetch(message.pid)
-        logger.info("worker=#{new_mold.nr} pid=#{new_mold.pid} promoted to a mold")
-        @children.update(message)
+        new_mold = @children.update(message)
+        logger.info("mold pid=#{new_mold.pid} gen=#{new_mold.generation} ready")
         old_molds.each do |old_mold|
-          logger.info("Terminating old mold pid=#{old_mold.pid}")
+          logger.info("Terminating old mold pid=#{old_mold.pid} gen=#{old_mold.generation}")
           old_mold.soft_kill(:QUIT)
         end
       else
@@ -306,17 +306,19 @@ module Pitchfork
 
     # Terminates all workers, but does not exit master process
     def stop(graceful = true)
+      @respawn = false
       wait_for_pending_workers
       self.listeners = []
-      limit = time_now + timeout
-      until @children.workers.empty? || time_now > limit
+      limit = Pitchfork.time_now + timeout
+      until @children.workers.empty? || Pitchfork.time_now > limit
         if graceful
           soft_kill_each_child(:QUIT)
         else
           kill_each_child(:TERM)
         end
-        sleep(0.1)
-        reap_all_workers
+        if monitor_loop(false) == StopIteration
+          return StopIteration
+        end
       end
       kill_each_child(:KILL)
       @promotion_lock.unlink
@@ -393,7 +395,7 @@ module Pitchfork
     # forcibly terminate all workers that haven't checked in in timeout seconds.  The timeout is implemented using an unlinked File
     def murder_lazy_workers
       next_sleep = @timeout - 1
-      now = time_now.to_i
+      now = Pitchfork.time_now(true)
       @children.workers.each do |worker|
         tick = worker.tick
         0 == tick and next # skip workers that haven't processed any clients
@@ -444,28 +446,11 @@ module Pitchfork
     def spawn_worker(worker, detach:)
       logger.info("worker=#{worker.nr} gen=#{worker.generation} spawning...")
 
-      pid = Pitchfork.clean_fork do
-        # We double fork so that the new worker is re-attached back
-        # to the master.
-        # This requires either PR_SET_CHILD_SUBREAPER which is exclusive to Linux 3.4
-        # or the master to be PID 1.
-        if detach && fork
-          exit
-        end
+      Pitchfork.fork_sibling do
         worker.pid = Process.pid
 
         after_fork_internal
         worker_loop(worker)
-        if worker.mold?
-          mold_loop(worker)
-        end
-        exit
-      end
-
-      if detach
-        # If we double forked, we need to wait(2) so that the middle
-        # process doesn't end up a zombie.
-        Process.wait(pid)
       end
 
       worker
@@ -475,12 +460,14 @@ module Pitchfork
       mold = Worker.new(nil)
       mold.create_socketpair!
       mold.pid = Pitchfork.clean_fork do
+        mold.pid = Process.pid
         @promotion_lock.try_lock
         mold.after_fork_in_child
         build_app!
         bind_listeners!
         mold_loop(mold)
       end
+      @promotion_lock.at_fork
       @children.register_mold(mold)
     end
 
@@ -494,7 +481,7 @@ module Pitchfork
 
         if REFORKING_AVAILABLE
           unless @children.mold&.spawn_worker(worker)
-            @logger.error("Failed to send a spawn_woker command")
+            @logger.error("Failed to send a spawn_worker command")
           end
         else
           spawn_worker(worker, detach: false)
@@ -529,17 +516,18 @@ module Pitchfork
       # If we're already in the middle of forking a new generation, we just continue
       return unless @children.mold
 
-      # We don't shutdown any outdated worker if any worker is already being spawned
-      # or a worker is exiting. Workers are only reforked one by one to minimize the
-      # impact on capacity.
-      # In the future we may want to use a dynamic limit, e.g. 10% of workers may be down at
-      # a time.
-      return if @children.pending_workers?
-      return if @children.workers.any?(&:exiting?)
-
-      if outdated_worker = @children.workers.find { |w| w.generation < @children.mold.generation }
-        logger.info("worker=#{outdated_worker.nr} pid=#{outdated_worker.pid} restarting")
-        outdated_worker.soft_kill(:QUIT)
+      # We don't shutdown any outdated worker if any worker is already being
+      # spawned or a worker is exiting. Only 10% of workers can be reforked at
+      # once to minimize the impact on capacity.
+      max_pending_workers = (worker_processes * 0.1).ceil
+      workers_to_restart = max_pending_workers - @children.restarting_workers_count
+
+      if workers_to_restart > 0
+        outdated_workers = @children.workers.select { |w| !w.exiting? && w.generation < @children.mold.generation }
+        outdated_workers.each do |worker|
+          logger.info("worker=#{worker.nr} pid=#{worker.pid} gen=#{worker.generation} restarting")
+          worker.soft_kill(:QUIT)
+        end
       end
     end
 
@@ -648,10 +636,10 @@ module Pitchfork
       (@queue_sigs - exit_sigs).each { |sig| trap(sig, nil) }
       trap(:CHLD, 'DEFAULT')
       @sig_queue.clear
-      proc_name "worker[#{worker.nr}] (gen:#{worker.generation})"
+      proc_name "(gen:#{worker.generation}) worker[#{worker.nr}]"
       @children = nil
 
-      after_fork.call(self, worker) # can drop perms and create listeners
+      after_worker_fork.call(self, worker) # can drop perms and create listeners
       LISTENERS.each { |sock| sock.close_on_exec = true }
 
       @config = nil
@@ -662,10 +650,10 @@ module Pitchfork
       readers
     end
 
-    def init_mold_process(worker)
-      proc_name "mold (gen: #{worker.generation})"
-      after_promotion.call(self, worker)
-      readers = [worker]
+    def init_mold_process(mold)
+      proc_name "(gen: #{mold.generation}) mold"
+      after_mold_fork.call(self, mold)
+      readers = [mold]
       trap(:QUIT) { nuke_listeners!(readers) }
       readers
     end
@@ -691,83 +679,99 @@ module Pitchfork
       ready = readers.dup
       @after_worker_ready.call(self, worker)
 
-      begin
-        worker.tick = time_now.to_i
-        while sock = ready.shift
-          # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
-          # but that will return false
-          client = sock.accept_nonblock(exception: false)
-          client = false if client == :wait_readable
-          if client
-            case client
-            when Message::PromoteWorker
-              if @promotion_lock.try_lock
-                logger.info("Refork asked by master, promoting ourselves")
-                worker.tick = time_now.to_i
-                return worker.promoted!
+      while readers[0]
+        begin
+          worker.tick = Pitchfork.time_now(true)
+          while sock = ready.shift
+            # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
+            # but that will return false
+            client = sock.accept_nonblock(exception: false)
+            client = false if client == :wait_readable
+            if client
+              case client
+              when Message::PromoteWorker
+                spawn_mold(worker.generation)
+              when Message
+                worker.update(client)
+              else
+                process_client(client)
+                worker.increment_requests_count
               end
-            when Message
-              worker.update(client)
-            else
-              process_client(client)
-              worker.increment_requests_count
+              worker.tick = Pitchfork.time_now(true)
             end
-            worker.tick = time_now.to_i
           end
-        end
 
-        # timeout so we can .tick and keep parent from SIGKILL-ing us
-        worker.tick = time_now.to_i
-        if @refork_condition && !worker.outdated?
-          if @refork_condition.met?(worker, logger)
-            if @promotion_lock.try_lock
-              logger.info("Refork condition met, promoting ourselves")
-              return worker.promote! # We've been promoted we can exit the loop
-            else
-              # TODO: if we couldn't acquire the lock, we should backoff the refork_condition to avoid hammering the lock
+          # timeout so we can .tick and keep parent from SIGKILL-ing us
+          worker.tick = Pitchfork.time_now(true)
+
+          if @refork_condition && !worker.outdated?
+            if @refork_condition.met?(worker, logger)
+              if spawn_mold(worker.generation)
+                logger.info("Refork condition met, promoting ourselves")
+              end
+              @refork_condition.backoff!
             end
           end
+
+          waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
+        rescue => e
+          Pitchfork.log_error(@logger, "listen loop error", e) if readers[0]
         end
+      end
+    end
+
+    def spawn_mold(current_generation)
+      return false unless @promotion_lock.try_lock
 
-        waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
-      rescue => e
-        Pitchfork.log_error(@logger, "listen loop error", e) if readers[0]
-      end while readers[0]
+      begin
+        Pitchfork.fork_sibling do
+          mold = Worker.new(nil, pid: Process.pid, generation: current_generation)
+          mold.promote!
+          mold.start_promotion(@control_socket[1])
+          mold_loop(mold)
+        end
+        true
+      ensure
+        @promotion_lock.at_fork # We let the spawned mold own the lock
+      end
     end
 
     def mold_loop(mold)
       readers = init_mold_process(mold)
       waiter = prep_readers(readers)
-      mold.declare_promotion(@control_socket[1])
       @promotion_lock.unlock
       ready = readers.dup
 
-      begin
-        mold.tick = time_now.to_i
-        while sock = ready.shift
-          # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
-          # but that will return false
-          message = sock.accept_nonblock(exception: false)
-          case message
-          when false
-            # no message, keep looping
-          when Message::SpawnWorker
-            begin
-              spawn_worker(Worker.new(message.nr, generation: mold.generation), detach: true)
-            rescue => error
-              raise BootFailure, error.message
+      mold.finish_promotion(@control_socket[1])
+
+      while readers[0]
+        begin
+          mold.tick = Pitchfork.time_now(true)
+          while sock = ready.shift
+            # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
+            # but that will return false
+            message = sock.accept_nonblock(exception: false)
+            case message
+            when false
+              # no message, keep looping
+            when Message::SpawnWorker
+              begin
+                spawn_worker(Worker.new(message.nr, generation: mold.generation), detach: true)
+              rescue => error
+                raise BootFailure, error.message
+              end
+            else
+              logger.error("Unexpected mold message #{message.inspect}")
             end
-          else
-            logger.error("Unexpected mold message #{message.inspect}")
           end
-        end
 
-        # timeout so we can .tick and keep parent from SIGKILL-ing us
-        mold.tick = time_now.to_i
-        waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
-      rescue => e
-        Pitchfork.log_error(@logger, "mold loop error", e) if readers[0]
-      end while readers[0]
+          # timeout so we can .tick and keep parent from SIGKILL-ing us
+          mold.tick = Pitchfork.time_now(true)
+          waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
+        rescue => e
+          Pitchfork.log_error(@logger, "mold loop error", e) if readers[0]
+        end
+      end
     end
 
     # delivers a signal to a worker and fails gracefully if the worker
@@ -820,9 +824,5 @@ module Pitchfork
       listeners.each { |addr| listen(addr) }
       raise ArgumentError, "no listeners" if LISTENERS.empty?
     end
-
-    def time_now
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
   end
 end

data/lib/pitchfork/message.rb

@@ -123,7 +123,8 @@ module Pitchfork
     SpawnWorker = Message.new(:nr)
     WorkerSpawned = Message.new(:nr, :pid, :generation, :pipe)
     PromoteWorker = Message.new(:generation)
-    WorkerPromoted = Message.new(:nr, :pid, :generation)
+    MoldSpawned = Message.new(:nr, :pid, :generation, :pipe)
+    MoldReady = Message.new(:nr, :pid, :generation)
 
     SoftKill = Message.new(:signum)
   end

data/lib/pitchfork/refork_condition.rb

@@ -5,17 +5,35 @@ module Pitchfork
     class RequestsCount
       def initialize(request_counts)
         @limits = request_counts
+        @backoff_until = nil
       end
 
       def met?(worker, logger)
         if limit = @limits.fetch(worker.generation) { @limits.last }
           if worker.requests_count >= limit
+            return false if backoff?
+
             logger.info("worker=#{worker.nr} pid=#{worker.pid} processed #{worker.requests_count} requests, triggering a refork")
             return true
           end
         end
         false
       end
+
+      def backoff?
+        return false if @backoff_until.nil?
+
+        if @backoff_until > Pitchfork.time_now
+          true
+        else
+          @backoff_until = nil
+          false
+        end
+      end
+
+      def backoff!(delay = 10.0)
+        @backoff_until = Pitchfork.time_now + delay
+      end
     end
   end
 end

data/lib/pitchfork/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Pitchfork
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
   module Const
     UNICORN_VERSION = '6.1.0'
   end