pitchfork 0.1.1 → 0.2.0

data/lib/pitchfork/http_server.rb

@@ -1,5 +1,6 @@
 # -*- encoding: binary -*-
 require 'pitchfork/pitchfork_http'
+require 'pitchfork/flock'
 
 module Pitchfork
   # This is the process manager of Pitchfork. This manages worker
@@ -9,11 +10,11 @@ module Pitchfork
   class HttpServer
     # :stopdoc:
     attr_accessor :app, :timeout, :worker_processes,
-                  :before_fork, :after_fork,
+                  :after_fork, :after_promotion,
                   :listener_opts, :children,
                   :orig_app, :config, :ready_pipe,
                   :default_middleware, :early_hints
-    attr_writer   :after_worker_exit, :after_worker_ready, :refork_condition, :mold_selector
+    attr_writer   :after_worker_exit, :after_worker_ready, :refork_condition
 
     attr_reader :logger
     include Pitchfork::SocketHelper
@@ -27,7 +28,6 @@ module Pitchfork
     NOOP = '.'
 
     REFORKING_AVAILABLE = Pitchfork::CHILD_SUBREAPER_AVAILABLE || Process.pid == 1
-    MAX_SLEEP = 1 # seconds
 
     # :startdoc:
     # This Hash is considered a stable interface and changing its contents
@@ -60,13 +60,15 @@ module Pitchfork
     # HttpServer.run.join to join the thread that's processing
     # incoming requests on the socket.
     def initialize(app, options = {})
+      @exit_status = 0
       @app = app
       @respawn = false
       @last_check = time_now
-      @default_middleware = true
+      @promotion_lock = Flock.new("pitchfork-promotion")
+
       options = options.dup
       @ready_pipe = options.delete(:ready_pipe)
-      @init_listeners = options[:listeners] ? options[:listeners].dup : []
+      @init_listeners = options[:listeners].dup || []
       options[:use_defaults] = true
       self.config = Pitchfork::Configurator.new(options)
       self.listener_opts = {}
@@ -104,7 +106,7 @@ module Pitchfork
     end
 
     # Runs the thing.  Returns self so you can run join on it
-    def start
+    def start(sync = true)
       Pitchfork.enable_child_subreaper # noop if not supported
 
       # This socketpair is used to wake us up from select(2) in #join when signals
@@ -120,7 +122,6 @@ module Pitchfork
       @queue_sigs.each { |sig| trap(sig) { @sig_queue << sig; awaken_master } }
       trap(:CHLD) { awaken_master }
 
-      bind_listeners!
       if REFORKING_AVAILABLE
         spawn_initial_mold
         wait_for_pending_workers
@@ -129,13 +130,17 @@ module Pitchfork
         end
       else
         build_app!
+        bind_listeners!
+        after_promotion.call(self, Worker.new(nil, pid: $$).promoted!)
       end
 
-      spawn_missing_workers
-      # We could just return here as we'd register them later in #join.
-      # However a good part of the test suite assumes #start only return
-      # once all initial workers are spawned.
-      wait_for_pending_workers
+      if sync
+        spawn_missing_workers
+        # We could just return here as we'd register them later in #join.
+        # However a good part of the test suite assumes #start only return
+        # once all initial workers are spawned.
+        wait_for_pending_workers
+      end
 
       self
     end
@@ -237,10 +242,19 @@ module Pitchfork
       end
       stop # gracefully shutdown all workers on our way out
       logger.info "master complete"
+      @exit_status
     end
 
     def monitor_loop(sleep = true)
       reap_all_workers
+
+      if REFORKING_AVAILABLE && @respawn && @children.molds.empty?
+        logger.info("No mold alive, shutting down")
+        @exit_status = 1
+        @sig_queue << :QUIT
+        @respawn = false
+      end
+
       case message = @sig_queue.shift
       when nil
         # avoid murdering workers after our master process (or the
@@ -253,13 +267,15 @@ module Pitchfork
         end
         if @respawn
           maintain_worker_count
-          automatically_refork_workers if REFORKING_AVAILABLE
+          restart_outdated_workers if REFORKING_AVAILABLE
         end
 
         master_sleep(sleep_time) if sleep
       when :QUIT # graceful shutdown
+        logger.info "QUIT received, starting graceful shutdown"
         return StopIteration
       when :TERM, :INT # immediate shutdown
+        logger.info "#{message} received, starting immediate shutdown"
         stop(false)
         return StopIteration
       when :USR2 # trigger a promotion
@@ -290,6 +306,7 @@ module Pitchfork
 
     # Terminates all workers, but does not exit master process
     def stop(graceful = true)
+      wait_for_pending_workers
       self.listeners = []
       limit = time_now + timeout
       until @children.workers.empty? || time_now > limit
@@ -302,6 +319,7 @@ module Pitchfork
         reap_all_workers
       end
       kill_each_child(:KILL)
+      @promotion_lock.unlink
     end
 
     def rewindable_input
@@ -333,8 +351,6 @@ module Pitchfork
 
     # wait for a signal handler to wake us up and then consume the pipe
     def master_sleep(sec)
-      sec = MAX_SLEEP if sec > MAX_SLEEP
-
       @control_socket[0].wait(sec) or return
       case message = @control_socket[0].recvmsg_nonblock(exception: false)
       when :wait_readable, NOOP
@@ -351,7 +367,7 @@ module Pitchfork
 
     # reaps all unreaped workers
     def reap_all_workers
-      begin
+      loop do
         wpid, status = Process.waitpid2(-1, Process::WNOHANG)
         wpid or return
         worker = @children.reap(wpid) and worker.close rescue nil
@@ -362,7 +378,7 @@ module Pitchfork
         end
       rescue Errno::ECHILD
         break
-      end while true
+      end
     end
 
     def listener_sockets
@@ -374,15 +390,6 @@ module Pitchfork
       listener_fds
     end
 
-    def close_sockets_on_exec(sockets)
-      (3..1024).each do |io|
-        next if sockets.include?(io)
-        io = IO.for_fd(io) rescue next
-        io.autoclose = false
-        io.close_on_exec = true
-      end
-    end
-
     # forcibly terminate all workers that haven't checked in in timeout seconds.  The timeout is implemented using an unlinked File
     def murder_lazy_workers
       next_sleep = @timeout - 1
@@ -413,17 +420,17 @@ module Pitchfork
       end
 
       unless @children.pending_promotion?
-        @children.refresh
-        if new_mold = @mold_selector.call(self)
+        if new_mold = @children.fresh_workers.first
           @children.promote(new_mold)
         else
-          logger.error("The mold select didn't return a candidate")
+          logger.error("No children at all???")
         end
       else
       end
     end
 
     def after_fork_internal
+      @promotion_lock.at_fork
       @control_socket[0].close_write # this is master-only, now
       @ready_pipe.close if @ready_pipe
       Pitchfork::Configurator::RACKUP.clear
@@ -435,9 +442,9 @@ module Pitchfork
     end
 
     def spawn_worker(worker, detach:)
-      before_fork.call(self, worker)
+      logger.info("worker=#{worker.nr} gen=#{worker.generation} spawning...")
 
-      pid = fork do
+      pid = Pitchfork.clean_fork do
         # We double fork so that the new worker is re-attached back
         # to the master.
         # This requires either PR_SET_CHILD_SUBREAPER which is exclusive to Linux 3.4
@@ -467,10 +474,11 @@ module Pitchfork
     def spawn_initial_mold
       mold = Worker.new(nil)
       mold.create_socketpair!
-      mold.pid = fork do
-        after_fork_internal
+      mold.pid = Pitchfork.clean_fork do
+        @promotion_lock.try_lock
         mold.after_fork_in_child
         build_app!
+        bind_listeners!
         mold_loop(mold)
       end
       @children.register_mold(mold)
@@ -484,9 +492,11 @@ module Pitchfork
         end
         worker = Pitchfork::Worker.new(worker_nr)
 
-        if !@children.mold || !@children.mold.spawn_worker(worker)
-          # If there's no mold, or the mold was somehow unreachable
-          # we fallback to spawning the missing workers ourselves.
+        if REFORKING_AVAILABLE
+          unless @children.mold&.spawn_worker(worker)
+            @logger.error("Failed to send a spawn_woker command")
+          end
+        else
           spawn_worker(worker, detach: false)
         end
         # We could directly register workers when we spawn from the
@@ -504,7 +514,7 @@ module Pitchfork
       while @children.pending_workers?
         master_sleep(0.5)
         if monitor_loop(false) == StopIteration
-          break
+          return StopIteration
         end
       end
     end
@@ -515,34 +525,21 @@ module Pitchfork
       @children.each_worker { |w| w.nr >= worker_processes and w.soft_kill(:QUIT) }
     end
 
-    def automatically_refork_workers
+    def restart_outdated_workers
       # If we're already in the middle of forking a new generation, we just continue
-      if @children.mold
-        # We don't shutdown any outdated worker if any worker is already being spawned
-        # or a worker is exiting. Workers are only reforked one by one to minimize the
-        # impact on capacity.
-        # In the future we may want to use a dynamic limit, e.g. 10% of workers may be down at
-        # a time.
-        return if @children.pending_workers?
-        return if @children.workers.any?(&:exiting?)
-
-        if outdated_worker = @children.workers.find { |w| w.generation < @children.mold.generation }
-          logger.info("worker=#{outdated_worker.nr} pid=#{outdated_worker.pid} restarting")
-          outdated_worker.soft_kill(:QUIT)
-          return # That's all folks
-        end
-      end
+      return unless @children.mold
 
-      # If all workers are alive and well, we can consider reforking a new generation
-      if @refork_condition
-        @children.refresh
-        if @refork_condition.met?(@children, logger)
-          logger.info("Refork condition met, scheduling a promotion")
-          unless @sig_queue.include?(:USR2)
-            @sig_queue << :USR2
-            awaken_master
-          end
-        end
+      # We don't shutdown any outdated worker if any worker is already being spawned
+      # or a worker is exiting. Workers are only reforked one by one to minimize the
+      # impact on capacity.
+      # In the future we may want to use a dynamic limit, e.g. 10% of workers may be down at
+      # a time.
+      return if @children.pending_workers?
+      return if @children.workers.any?(&:exiting?)
+
+      if outdated_worker = @children.workers.find { |w| w.generation < @children.mold.generation }
+        logger.info("worker=#{outdated_worker.nr} pid=#{outdated_worker.pid} restarting")
+        outdated_worker.soft_kill(:QUIT)
       end
     end
 
@@ -572,22 +569,11 @@ module Pitchfork
     end
 
     def e103_response_write(client, headers)
-      response = if @request.response_start_sent
-        "103 Early Hints\r\n"
-      else
-        "HTTP/1.1 103 Early Hints\r\n"
-      end
-
-      headers.each_pair do |k, vs|
-        next if !vs || vs.empty?
-        values = vs.to_s.split("\n".freeze)
-        values.each do |v|
-          response << "#{k}: #{v}\r\n"
-        end
-      end
-      response << "\r\n".freeze
-      response << "HTTP/1.1 ".freeze if @request.response_start_sent
-      client.write(response)
+      rss = @request.response_start_sent
+      buf = rss ? "103 Early Hints\r\n" : "HTTP/1.1 103 Early Hints\r\n"
+      headers.each { |key, value| append_header(buf, key, value) }
+      buf << (rss ? "\r\nHTTP/1.1 ".freeze : "\r\n".freeze)
+      client.write(buf)
     end
 
     def e100_response_write(client, env)
@@ -678,6 +664,7 @@ module Pitchfork
 
     def init_mold_process(worker)
       proc_name "mold (gen: #{worker.generation})"
+      after_promotion.call(self, worker)
       readers = [worker]
       trap(:QUIT) { nuke_listeners!(readers) }
       readers
@@ -713,6 +700,12 @@ module Pitchfork
           client = false if client == :wait_readable
           if client
             case client
+            when Message::PromoteWorker
+              if @promotion_lock.try_lock
+                logger.info("Refork asked by master, promoting ourselves")
+                worker.tick = time_now.to_i
+                return worker.promoted!
+              end
             when Message
               worker.update(client)
             else
@@ -721,11 +714,21 @@ module Pitchfork
             end
             worker.tick = time_now.to_i
           end
-          return if worker.mold? # We've been promoted we can exit the loop
         end
 
         # timeout so we can .tick and keep parent from SIGKILL-ing us
         worker.tick = time_now.to_i
+        if @refork_condition && !worker.outdated?
+          if @refork_condition.met?(worker, logger)
+            if @promotion_lock.try_lock
+              logger.info("Refork condition met, promoting ourselves")
+              return worker.promote! # We've been promoted we can exit the loop
+            else
+              # TODO: if we couldn't acquire the lock, we should backoff the refork_condition to avoid hammering the lock
+            end
+          end
+        end
+
         waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
       rescue => e
         Pitchfork.log_error(@logger, "listen loop error", e) if readers[0]
@@ -735,10 +738,9 @@ module Pitchfork
     def mold_loop(mold)
       readers = init_mold_process(mold)
       waiter = prep_readers(readers)
-      mold.acknowlege_promotion(@control_socket[1])
-
+      mold.declare_promotion(@control_socket[1])
+      @promotion_lock.unlock
       ready = readers.dup
-      # TODO: mold ready callback?
 
       begin
         mold.tick = time_now.to_i
@@ -750,7 +752,11 @@ module Pitchfork
           when false
             # no message, keep looping
           when Message::SpawnWorker
-            spawn_worker(Worker.new(message.nr, generation: mold.generation), detach: true)
+            begin
+              spawn_worker(Worker.new(message.nr, generation: mold.generation), detach: true)
+            rescue => error
+              raise BootFailure, error.message
+            end
           else
             logger.error("Unexpected mold message #{message.inspect}")
           end
@@ -760,7 +766,7 @@ module Pitchfork
         mold.tick = time_now.to_i
         waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
       rescue => e
-        Pitchfork.log_error(@logger, "listen loop error", e) if readers[0]
+        Pitchfork.log_error(@logger, "mold loop error", e) if readers[0]
       end while readers[0]
     end
 

data/lib/pitchfork/refork_condition.rb

@@ -7,9 +7,9 @@ module Pitchfork
         @limits = request_counts
       end
 
-      def met?(children, logger)
-        if limit = @limits[children.last_generation]
-          if worker = children.fresh_workers.find { |w| w.requests_count >= limit }
+      def met?(worker, logger)
+        if limit = @limits.fetch(worker.generation) { @limits.last }
+          if worker.requests_count >= limit
             logger.info("worker=#{worker.nr} pid=#{worker.pid} processed #{worker.requests_count} requests, triggering a refork")
             return true
           end

data/lib/pitchfork/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Pitchfork
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
   module Const
     UNICORN_VERSION = '6.1.0'
   end

data/lib/pitchfork/worker.rb

@@ -7,14 +7,13 @@ module Pitchfork
   # releases of pitchfork.  Knowledge of this class is generally not
   # not needed for most users of pitchfork.
   #
-  # Some users may want to access it in the before_fork/after_fork hooks.
+  # Some users may want to access it in the after_promotion/after_fork hooks.
   # See the Pitchfork::Configurator RDoc for examples.
   class Worker
     # :stopdoc:
     EXIT_SIGNALS = [:QUIT, :TERM]
-    @generation = 0
     attr_accessor :nr, :pid, :generation
-    attr_reader :master
+    attr_reader :master, :requests_count
 
     def initialize(nr, pid: nil, generation: 0)
       @nr = nr
@@ -23,6 +22,7 @@ module Pitchfork
       @mold = false
       @to_io = @master = nil
       @exiting = false
+      @requests_count = 0
       if nr
         build_raindrops(nr)
       else
@@ -42,13 +42,17 @@ module Pitchfork
       @exiting
     end
 
+    def outdated?
+      CURRENT_GENERATION_DROP[0] > @generation
+    end
+
     def update(message)
       message.class.members.each do |member|
         send("#{member}=", message.public_send(member))
       end
 
       case message
-      when Message::WorkerPromoted, Message::PromoteWorker
+      when Message::WorkerPromoted
         promoted!
       end
     end
@@ -60,9 +64,10 @@ module Pitchfork
       @master.close
     end
 
-    def acknowlege_promotion(control_socket)
+    def declare_promotion(control_socket)
       message = Message::WorkerPromoted.new(@nr, Process.pid, generation)
       control_socket.sendmsg(message)
+      CURRENT_GENERATION_DROP[0] = @generation
     end
 
     def promote(generation)
@@ -73,11 +78,17 @@ module Pitchfork
       send_message_nonblock(Message::SpawnWorker.new(new_worker.nr))
     end
 
+    def promote!
+      @generation += 1
+      promoted!
+    end
+
     def promoted!
       @mold = true
       @nr = nil
       @drop_offset = 0
       @tick_drop = MOLD_DROP
+      self
     end
 
     def mold?
@@ -169,15 +180,11 @@ module Pitchfork
     end
 
     def reset
-      @requests_drop[@drop_offset] = 0
-    end
-
-    def requests_count
-      @requests_drop[@drop_offset]
+      @requests_count = 0
     end
 
-    def increment_requests_count
-      @requests_drop.incr(@drop_offset)
+    def increment_requests_count(by = 1)
+      @requests_count += by
     end
 
     # called in both the master (reaping worker) and worker (SIGQUIT handler)
@@ -215,9 +222,9 @@ module Pitchfork
     end
 
     MOLD_DROP = Raindrops.new(1)
+    CURRENT_GENERATION_DROP = Raindrops.new(1)
     PER_DROP = Raindrops::PAGE_SIZE / Raindrops::SIZE
     TICK_DROPS = []
-    REQUEST_DROPS = []
 
     class << self
       # Since workers are created from another process, we have to
@@ -228,7 +235,6 @@ module Pitchfork
       def preallocate_drops(workers_count)
         0.upto(workers_count / PER_DROP) do |i|
           TICK_DROPS[i] = Raindrops.new(PER_DROP)
-          REQUEST_DROPS[i] = Raindrops.new(PER_DROP)
         end
       end
     end
@@ -237,8 +243,7 @@ module Pitchfork
       drop_index = drop_nr / PER_DROP
       @drop_offset = drop_nr % PER_DROP
       @tick_drop = TICK_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
-      @requests_drop = REQUEST_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
-      @tick_drop[@drop_offset] = @requests_drop[@drop_offset] = 0
+      @tick_drop[@drop_offset] = 0
     end
   end
 end

data/lib/pitchfork.rb

@@ -57,40 +57,15 @@ module Pitchfork
         Object.const_get(File.basename(ru, '.rb').capitalize)
       end
 
-      if $DEBUG
-        require 'pp'
-        pp({ :inner_app => inner_app })
-      end
-
-      return inner_app unless server.default_middleware
-
-      middleware = { # order matters
-        ContentLength: nil,
-        Chunked: nil,
-        CommonLogger: [ $stderr ],
-        ShowExceptions: nil,
-        Lint: nil,
-        TempfileReaper: nil,
-      }
-
-      # return value, matches rackup defaults based on env
-      # Pitchfork does not support persistent connections, but Rainbows!
-      # and Zbatery both do.  Users accustomed to the Rack::Server default
-      # middlewares will need ContentLength/Chunked middlewares.
       case ENV["RACK_ENV"]
       when "development"
-      when "deployment"
-        middleware.delete(:ShowExceptions)
-        middleware.delete(:Lint)
+        Rack::Builder.new do
+          use(Rack::Lint)
+          run inner_app
+        end.to_app
       else
-        return inner_app
+        inner_app
       end
-      Rack::Builder.new do
-        middleware.each do |m, args|
-          use(Rack.const_get(m), *args) if Rack.const_defined?(m)
-        end
-        run inner_app
-      end.to_app
     end
   end
 
@@ -146,13 +121,32 @@ module Pitchfork
       raise
     end
   end
+
+  def self.clean_fork(&block)
+    # We fork from a thread to start with a clean stack.
+    # If we didn't the base stack would grow after each refork
+    # putting an effective limit on the number of generations.
+    parent_thread = Thread.current
+    Thread.new do
+      current_thread = Thread.current
+      # We copy over any thread state it might have
+      parent_thread.keys.each do |key|
+        current_thread[key] = parent_thread[key]
+      end
+      parent_thread.thread_variables.each do |variable|
+        current_thread.thread_variable_set(variable, parent_thread.thread_variable_get(variable))
+      end
+
+      fork(&block)
+    end.value
+  end
   # :startdoc:
 end
 # :enddoc:
 
 %w(
   const socket_helper stream_input tee_input mem_info children message http_parser
-  refork_condition mold_selector configurator tmpio http_response worker http_server
+  refork_condition configurator tmpio http_response worker http_server
 ).each do |s|
   require_relative "pitchfork/#{s}"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pitchfork
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-11-05 00:00:00.000000000 Z
+date: 2023-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: raindrops
@@ -96,13 +96,13 @@ files:
 - lib/pitchfork/children.rb
 - lib/pitchfork/configurator.rb
 - lib/pitchfork/const.rb
+- lib/pitchfork/flock.rb
 - lib/pitchfork/http_parser.rb
 - lib/pitchfork/http_response.rb
 - lib/pitchfork/http_server.rb
 - lib/pitchfork/launcher.rb
 - lib/pitchfork/mem_info.rb
 - lib/pitchfork/message.rb
-- lib/pitchfork/mold_selector.rb
 - lib/pitchfork/preread_input.rb
 - lib/pitchfork/refork_condition.rb
 - lib/pitchfork/select_waiter.rb
@@ -133,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: Rack HTTP server for fast clients and Unix

data/lib/pitchfork/mold_selector.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Pitchfork
-  module MoldSelector
-    class LeastSharedMemory
-      def call(server)
-        workers = server.children.fresh_workers
-        if workers.empty?
-          server.logger.info("No current generation workers yet")
-          return
-        end
-        candidate = workers.shift
-
-        workers.each do |worker|
-          if worker.meminfo.shared_memory < candidate.meminfo.shared_memory
-            # We suppose that a worker with a lower amount of shared memory
-            # has warmed up more caches & such, hence is closer to stabilize
-            # making it a better candidate.
-            candidate = worker
-          end
-        end
-        parent_meminfo = server.children.mold&.meminfo || MemInfo.new(Process.pid)
-        cow_efficiency = candidate.meminfo.cow_efficiency(parent_meminfo)
-        server.logger.info("worker=#{candidate.nr} pid=#{candidate.pid} selected as new mold shared_memory_kb=#{candidate.meminfo.shared_memory} cow=#{cow_efficiency.round(1)}%")
-        candidate
-      end
-    end
-  end
-end