pitchfork 0.17.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b22167ab418be7e6f7ad8a8c66177582b80e611423531f7f96f132688f684c8d
-  data.tar.gz: 036fbed35ccdece6c17b26f0f4011b78efcc33e4344e6fbe94fcb5ee170d52d3
+  metadata.gz: 735c4d6910ec05b65ae12e6e13f306110d55102d4ec41d93be14033dbda00f6f
+  data.tar.gz: 1452376664a49ff6c81c4c29e04345ef93b891f68d4d87d53e40473c6ed5e67b
 SHA512:
-  metadata.gz: cd7bd9b5ef42730eb970b368d18fc188834e65a8a7104666cf320d6ef71e629196dbab2ef081a11c988b5232de9084d7debe732ee306524dc503d7464f33823f
-  data.tar.gz: 53db8ad62bfdd0e21d8e01734427a9cc59a3594438b61616a2c0151dc563d5d2bc84521b1085ba797821598a8d35c9f95db12d50c3a909dfe03bc7834404c49b
+  metadata.gz: 4d9d99a1092c508806c2b7877930637c54c7a0e79aeb3cb9d74b22f9fc2a25caa65b26a5d8f73c9a78ecdc072e7da285c1f1fd1b1d025b096acb652bcf46db90
+  data.tar.gz: 05b55878eb9fe2c7d8af70429789f31aaa4ec5c30e025fd5fb4a6d6e3934256fe9c66dce75fbc98f2e0d596f56874a5ccc06578923c9d15f7811e768a02ecf4c

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Unreleased
 
+# 0.18.0
+
+- Fix a regression introduced in `0.17.0` causing `before_worker_exit` to often not be called.
+- Implemented the `max_consecutive_spawn_errors` configuration. Purely opt-in for now.
+- Properly handle magic comments in `config.ru`.
+- Execute `rack.response_finished` callbacks in reverse order of registration
+
 # 0.17.0
 
 - Improve `Pitchfork::Info#live_workers_count` to be more accurate.

data/docs/CONFIGURATION.md

@@ -542,6 +542,37 @@ By default automatic reforking isn't enabled.
 
 Make sure to read the [fork safety guide](FORK_SAFETY.md) before enabling reforking.
 
+### `refork_max_unavailable`
+
+```ruby
+refork_max_unavailable 5
+```
+
+Sets the number of workers than can be restarted at a given time.
+
+When a new generation is created, workers from the old generation are rolled out progressively
+and replaced by fresh workers from the new generation.
+
+For instance `refork_max_unavailable 5` means 5 workers may be unavailable during the rollout phase.
+
+The default is `(workers_processes * 0.1).ceil`, or `10%` rounded up.
+
+### `max_consecutive_spawn_errors`
+
+```ruby
+max_consecutive_spawn_errors 5
+```
+
+Sets the number of consecutive failures of spawning new workers that trigger a shutdown.
+
+Whenever a newly spawned worker fail to acheive readiness, either because it crashes or because
+it times out before becoming ready, a counter is incremented. Whenever a worker successfully
+reach readiness, the counter is reset.
+
+This can be useful to better handle issues with slow or broken `after_worker_fork` callbacks.
+
+The default is `nil`, which means the behavior is disabled.
+
 ## Rack Features
 
 ### `early_hints`

data/lib/pitchfork/configurator.rb

@@ -75,7 +75,7 @@ module Pitchfork
       :client_body_buffer_size => Pitchfork::Const::MAX_BODY,
       :before_service_worker_ready => nil,
       :before_service_worker_exit => nil,
-      :setpgid => false,
+      :setpgid => true,
     }
     #:startdoc:
 
@@ -216,6 +216,14 @@ module Pitchfork
       set_int(:worker_processes, nr, 1)
     end
 
+    def refork_max_unavailable(max)
+      set_int(:refork_max_unavailable, max, 1)
+    end
+
+    def max_consecutive_spawn_errors(max)
+      set_int(:max_consecutive_spawn_errors, max, 1)
+    end
+
     def early_hints(bool)
       set_bool(:early_hints, bool)
     end

data/lib/pitchfork/http_server.rb

@@ -82,7 +82,8 @@ module Pitchfork
                   :listener_opts, :children,
                   :orig_app, :config, :ready_pipe, :early_hints, :setpgid
     attr_writer   :after_worker_exit, :before_worker_exit, :after_worker_ready, :after_request_complete,
-                  :refork_condition, :after_worker_timeout, :after_worker_hard_timeout, :after_monitor_ready
+                  :refork_condition, :after_worker_timeout, :after_worker_hard_timeout, :after_monitor_ready, :refork_max_unavailable,
+                  :max_consecutive_spawn_errors
 
     attr_reader :logger
     include Pitchfork::SocketHelper
@@ -103,6 +104,9 @@ module Pitchfork
       @exit_status = 0
       @app = app
       @respawn = false
+      @refork_max_unavailable = nil
+      @consecutive_spawn_errors = 0
+      @max_consecutive_spawn_errors = nil
       @last_check = Pitchfork.time_now
       @promotion_lock = Flock.new("pitchfork-promotion")
       Info.keep_io(@promotion_lock)
@@ -330,6 +334,13 @@ module Pitchfork
         # machine) comes out of suspend/hibernation
         if (@last_check + @timeout) >= (@last_check = Pitchfork.time_now)
           sleep_time = murder_lazy_workers
+          if @max_consecutive_spawn_errors && @consecutive_spawn_errors > @max_consecutive_spawn_errors && !SharedMemory.shutting_down?
+            logger.fatal("#{@consecutive_spawn_errors} consecutive failures to spawn children, aborting - broken after_worker_fork callback?")
+            @exit_status = 1
+            SharedMemory.shutting_down!
+            stop(false)
+            return StopIteration
+          end
         else
           sleep_time = @timeout/2.0 + 1
           @logger.debug("waiting #{sleep_time}s after suspend/hibernation")
@@ -371,6 +382,7 @@ module Pitchfork
         new_service = @children.update(message)
         logger.info("#{new_service.to_log} spawned")
       when Message::MoldReady
+        @consecutive_spawn_errors = 0
         old_molds = @children.molds
         new_mold = @children.update(message)
         logger.info("#{new_mold.to_log} ready")
@@ -378,6 +390,8 @@ module Pitchfork
           logger.info("Terminating old #{old_mold.to_log}")
           old_mold.soft_kill(:TERM)
         end
+      when Message::WorkerReady, Message::ServiceReady
+        @consecutive_spawn_errors = 0
       else
         logger.error("Unexpected message in sig_queue #{message.inspect}")
         logger.error(@sig_queue.inspect)
@@ -467,6 +481,10 @@ module Pitchfork
 
     private
 
+    def refork_max_unavailable
+      @refork_max_unavailable ||= (worker_processes * 0.1).ceil
+    end
+
     # wait for a signal handler to wake us up and then consume the pipe
     def monitor_sleep(sec)
       @control_socket[0].wait(sec) or return
@@ -490,6 +508,9 @@ module Pitchfork
         wpid or return
         worker = @children.reap(wpid) and worker.close rescue nil
         if worker
+          unless worker.ready?
+            @consecutive_spawn_errors += 1
+          end
           @after_worker_exit.call(self, worker, status)
         else
           logger.info("reaped unknown subprocess #{status.inspect}")
@@ -549,6 +570,7 @@ module Pitchfork
       end
 
       logger.error "#{child.to_log} timed out, killing"
+      @consecutive_spawn_errors += 1 unless child.ready?
       @children.hard_kill(@timeout_signal.call(child.pid), child) # take no prisoners for hard timeout violations
     end
 
@@ -613,6 +635,7 @@ module Pitchfork
         end
       end
 
+      service.notify_ready(@control_socket[1])
       proc_name status: "ready"
 
       while readers[0]
@@ -750,8 +773,7 @@ module Pitchfork
       # We don't shutdown any outdated worker if any worker is already being
       # spawned or a worker is exiting. Only 10% of workers can be reforked at
       # once to minimize the impact on capacity.
-      max_pending_workers = (worker_processes * 0.1).ceil
-      workers_to_restart = max_pending_workers - @children.restarting_workers_count
+      workers_to_restart = refork_max_unavailable - @children.restarting_workers_count
 
       if service = @children.service
         if service.outdated?
@@ -871,7 +893,7 @@ module Pitchfork
       env
     ensure
       if env
-        env["rack.response_finished"].each do |callback|
+        env["rack.response_finished"].reverse_each do |callback|
           if callback.arity == 0
             callback.call
           else
@@ -926,14 +948,13 @@ module Pitchfork
     end
 
     def init_service_process(service)
-      proc_name role: "(gen:#{service.generation}) mold", status: "init"
+      proc_name role: "(gen:#{service.generation}) service", status: "init"
       LISTENERS.each(&:close).clear # Don't appear as listening to incoming requests
       service.register_to_monitor(@control_socket[1])
       readers = [service]
       trap(:QUIT) { nuke_listeners!(readers) }
       trap(:TERM) { nuke_listeners!(readers) }
       trap(:INT) { nuke_listeners!(readers); exit!(0) }
-      proc_name role: "(gen:#{service.generation}) service", status: "ready"
       readers
     end
 
@@ -944,7 +965,6 @@ module Pitchfork
       trap(:QUIT) { nuke_listeners!(readers) }
       trap(:TERM) { nuke_listeners!(readers) }
       trap(:INT) { nuke_listeners!(readers); exit!(0) }
-      proc_name role: "(gen:#{mold.generation}) mold", status: "ready"
       readers
     end
 
@@ -969,10 +989,9 @@ module Pitchfork
       ready = readers.dup
       @after_worker_ready.call(self, worker)
 
+      worker.notify_ready(@control_socket[1])
       proc_name status: "ready"
 
-      worker.ready = true
-
       while readers[0]
         begin
           worker.update_deadline(@timeout)
@@ -1054,6 +1073,8 @@ module Pitchfork
       ready = readers.dup
 
       mold.finish_promotion(@control_socket[1])
+      mold.ready = true
+      proc_name status: "ready"
 
       while readers[0]
         begin
@@ -1165,8 +1186,6 @@ module Pitchfork
     FORK_TIMEOUT = 5
 
     def fork_sibling(role, &block)
-      reset_signal_handlers
-
       if REFORKING_AVAILABLE
         r, w = Pitchfork::Info.keep_ios(IO.pipe)
         # We double fork so that the new worker is re-attached back
@@ -1190,6 +1209,8 @@ module Pitchfork
             raise ForkFailure, "fork_sibling didn't succeed in #{FORK_TIMEOUT} seconds"
           end
         else # first child
+          reset_signal_handlers
+
           r.close
           Process.setproctitle("<pitchfork fork_sibling(#{role})>")
           pid = Pitchfork.clean_fork(setpgid: setpgid) do

data/lib/pitchfork/message.rb

@@ -124,6 +124,7 @@ module Pitchfork
   class Message
     SpawnWorker = new(:nr)
     WorkerSpawned = new(:nr, :pid, :generation, :pipe)
+    WorkerReady = new(:nr, :pid, :generation)
     PromoteWorker = new(:generation)
 
     MoldSpawned = new(:nr, :pid, :generation, :pipe)
@@ -131,6 +132,7 @@ module Pitchfork
 
     SpawnService = new(:_) # Struct.new requires at least 1 member on Ruby < 3.3
     ServiceSpawned = new(:pid, :generation, :pipe)
+    ServiceReady = new(:pid, :generation)
 
     SoftKill = new(:signum)
   end

data/lib/pitchfork/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Pitchfork
-  VERSION = "0.17.0"
+  VERSION = "0.18.0"
   module Const
     UNICORN_VERSION = '6.1.0'
   end

data/lib/pitchfork/worker.rb

@@ -67,12 +67,25 @@ module Pitchfork
     end
 
     def finish_promotion(control_socket)
+      SharedMemory.current_generation = @generation
       message = Message::MoldReady.new(@nr, @pid, generation)
       control_socket.sendmsg(message)
-      SharedMemory.current_generation = @generation
       @state_drop = SharedMemory.mold_state
     end
 
+    def notify_ready(control_socket)
+      self.ready = true
+      message = if worker?
+        Message::WorkerReady.new(@nr, @pid, @generation)
+      elsif service?
+        Message::ServiceReady.new(@pid, @generation)
+      else
+        raise "Unexpected child type"
+      end
+
+      control_socket.sendmsg(message)
+    end
+
     def promote(generation)
       send_message_nonblock(Message::PromoteWorker.new(generation))
     end
@@ -106,6 +119,10 @@ module Pitchfork
       false
     end
 
+    def worker?
+      !mold? && !service?
+    end
+
     def to_io # IO.select-compatible
       @to_io.to_io
     end

data/lib/pitchfork.rb

@@ -87,7 +87,12 @@ module Pitchfork
         when /\.ru$/
           raw = File.read(ru)
           raw.sub!(/^__END__\n.*/, '')
-          eval("Rack::Builder.new {(\n#{raw}\n)}.to_app", TOPLEVEL_BINDING, ru)
+          lines = raw.lines
+          trailing_comments_index = lines.index { |line| !line.start_with?('#') }
+          prelude = lines[0...trailing_comments_index].join
+          raw = lines[trailing_comments_index..-1].join
+
+          eval("#{prelude}\nRack::Builder.new do\n#{raw}\nend.to_app\n", TOPLEVEL_BINDING, ru)
         else
           require ru
           Object.const_get(File.basename(ru, '.rb').capitalize)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pitchfork
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-24 00:00:00.000000000 Z
+date: 2025-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack