pitchfork 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb4e22969b9f2c38717f0cfa7a3c966995814156a17589bc06bdc609b7ad6e32
-  data.tar.gz: dbf7833c26ef94962abbd71d66c63e40dd0f7f405ee82951723ad3b4cbfa864f
+  metadata.gz: 0f23c677bd64eb8ca7a040cb2d41dab4047e05ae849da16af8e537748bfb6dbe
+  data.tar.gz: 890bac5b67c23d6f0aacf0d9130999cf69ac0bc5a2f1ea06bdd142d67c0606ac
 SHA512:
-  metadata.gz: 0f0c029a01bc999d90421fe0ed77f76c6f1e56a9f3ec8eaa4ab6722f40eb0ee7f9aa0f004044e4c97e93a593cdba0d7a6bab01ef6b075440c76012f05e5cccd4
-  data.tar.gz: 4ae4d319ffd2acbf99ad29156622510c951ca095c20074f8415376d6e217be1b959e68e3245590cce62de66f3f8af8f0aae469fec842865dd4dcfe1b060e3db9
+  metadata.gz: 39c33bbbe865ba22bebc8b6b1e823f71d0ba6b2787f06af26f0ffe3d4251c781850826fcbcdf1dc13b1734919770e35dc57815d2de057f18d97ba354f247a3e7
+  data.tar.gz: 94e5ae39e129af1fd950c9a77e094bac5bda586f7358115fd8cf25bb3c6ee2079bb4654ebd4e6ed5e9ff498b461559da07d7b44acc0147fcac5c9a4695b7c62c

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Unreleased
 
+# 0.10.0
+
+- Include requests count in workers proctitle.
+
 # 0.9.0
 
 - Implement `spawn_timeout` to protect against bugs causing workers to get stuck before they reach ready state.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pitchfork (0.9.0)
+    pitchfork (0.10.0)
       rack (>= 2.0)
       raindrops (~> 0.7)
 

data/docs/REFORKING.md

@@ -66,7 +66,10 @@ PID   COMMAND
 105       \_ pitchfork (gen:0) worker[3]
 ```
 
-When a reforking is triggered, one of the workers is selected to fork a new `mold`.
+As the diagram shows, while workers are forked from the mold, they become children of the master process.
+We'll see how does that work [later](#forking-sibling-processes).
+
+When a reforking is triggered, one of the workers is selected to fork a new `mold`:
 
 ```
 PID   COMMAND
@@ -79,6 +82,9 @@ PID   COMMAND
 105       \_ pitchfork (gen:1) mold
 ```
 
+Again, while the mold was forked from a worker, it becomes a child of the master process.
+We'll see how does that work [later](#forking-sibling-processes).
+
 When that new mold is ready, `pitchfork` terminates the old mold and starts a slow rollout of older workers and replace them with fresh workers
 forked from the mold:
 
@@ -104,7 +110,7 @@ PID   COMMAND
 
 etc.
 
-### Forking Sibling Processes 
+### Forking Sibling Processes
 
 Normally on unix systems, when calling `fork(2)`, the newly created process is a child of the original one, so forking from the mold should create
 a process tree such as:
@@ -119,5 +125,8 @@ PID   COMMAND
 However the `pitchfork` master process registers itself as a "child subreaper" via [`PR_SET_CHILD_SUBREAPER`](https://man7.org/linux/man-pages/man2/prctl.2.html).
 This means any descendant process that is orphaned will be re-parented as a child of the master rather than a child of the init process (pid 1).
 
-With this in mind, the mold fork twice to create an orphaned process that will get re-attached to the master, effectively forking a sibling rather than a child.
-The need for `PR_SET_CHILD_SUBREAPER` is the main reason why reforking is only available on Linux. 
+With this in mind, the mold forks twice to create an orphaned process that will get re-attached to the master,
+effectively forking a sibling rather than a child. Similarly, workers do the same when forking new molds.
+This technique eases killing previous generations of molds and workers.
+
+The need for `PR_SET_CHILD_SUBREAPER` is the main reason why reforking is only available on Linux.

data/lib/pitchfork/children.rb

@@ -107,6 +107,33 @@ module Pitchfork
       @workers.each_value(&block)
     end
 
+    def soft_kill_all(sig)
+      each do |child|
+        child.soft_kill(sig)
+      end
+    end
+
+    def hard_kill(sig, child)
+      child.hard_kill(sig)
+    rescue Errno::ESRCH
+      reap(child.pid)
+      child.close
+    end
+
+    def hard_kill_all(sig)
+      each do |child|
+        hard_kill(sig, child)
+      end
+    end
+
+    def hard_timeout(child)
+      child.hard_timeout!
+    rescue Errno::ESRCH
+      reap(child.pid)
+      child.close
+      true
+    end
+
     def workers
       @workers.values
     end

data/lib/pitchfork/http_server.rb

@@ -396,15 +396,15 @@ module Pitchfork
       limit = Pitchfork.time_now + timeout
       until @children.workers.empty? || Pitchfork.time_now > limit
         if graceful
-          soft_kill_each_child(:TERM)
+          @children.soft_kill_all(:TERM)
         else
-          kill_each_child(:INT)
+          @children.hard_kill_all(:INT)
         end
         if monitor_loop(false) == StopIteration
           return StopIteration
         end
       end
-      kill_each_child(:KILL)
+      @children.hard_kill_all(:KILL)
       @promotion_lock.unlink
     end
 
@@ -506,25 +506,28 @@ module Pitchfork
           next
         else # worker is out of time
           next_sleep = 0
-          if worker.mold?
-            logger.error "mold pid=#{worker.pid} timed out, killing"
-          else
-            logger.error "worker=#{worker.nr} pid=#{worker.pid} timed out, killing"
-          end
+          hard_timeout(worker)
+        end
+      end
 
-          if @after_worker_hard_timeout
-            begin
-              @after_worker_hard_timeout.call(self, worker)
-            rescue => error
-              Pitchfork.log_error(@logger, "after_worker_hard_timeout callback", error)
-            end
-          end
+      next_sleep <= 0 ? 1 : next_sleep
+    end
 
-          kill_worker(:KILL, worker.pid) # take no prisoners for hard timeout violations
+    def hard_timeout(worker)
+      if @after_worker_hard_timeout
+        begin
+          @after_worker_hard_timeout.call(self, worker)
+        rescue => error
+          Pitchfork.log_error(@logger, "after_worker_hard_timeout callback", error)
         end
       end
 
-      next_sleep <= 0 ? 1 : next_sleep
+      if worker.mold?
+        logger.error "mold pid=#{worker.pid} timed out, killing"
+      else
+        logger.error "worker=#{worker.nr} pid=#{worker.pid} timed out, killing"
+      end
+      @children.hard_timeout(worker) # take no prisoners for hard timeout violations
     end
 
     def trigger_refork
@@ -697,12 +700,12 @@ module Pitchfork
 
     # once a client is accepted, it is processed in its entirety here
     # in 3 easy steps: read request, call app, write app response
-    def process_client(client, timeout_handler)
+    def process_client(client, worker, timeout_handler)
       env = nil
       @request = Pitchfork::HttpParser.new
       env = @request.read(client)
 
-      proc_name status: "processing: #{env["PATH_INFO"]}"
+      proc_name status: "requests: #{worker.requests_count}, processing: #{env["PATH_INFO"]}"
 
       timeout_handler.rack_env = env
       env["pitchfork.timeout"] = timeout_handler
@@ -835,7 +838,7 @@ module Pitchfork
               when Message
                 worker.update(client)
               else
-                request_env = process_client(client, prepare_timeout(worker))
+                request_env = process_client(client, worker, prepare_timeout(worker))
                 @after_request_complete&.call(self, worker, request_env)
                 worker.increment_requests_count
               end
@@ -848,6 +851,7 @@ module Pitchfork
 
           if @refork_condition && Info.fork_safe? && !worker.outdated?
             if @refork_condition.met?(worker, logger)
+              proc_name status: "requests: #{worker.requests_count}, spawning mold"
               if spawn_mold(worker.generation)
                 logger.info("Refork condition met, promoting ourselves")
               end
@@ -855,7 +859,7 @@ module Pitchfork
             end
           end
 
-          proc_name status: "waiting"
+          proc_name status: "requests: #{worker.requests_count}, waiting"
           waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
         rescue => e
           Pitchfork.log_error(@logger, "listen loop error", e) if readers[0]
@@ -930,15 +934,6 @@ module Pitchfork
       worker = @children.reap(wpid) and worker.close rescue nil
     end
 
-    # delivers a signal to each worker
-    def kill_each_child(signal)
-      @children.each { |w| kill_worker(signal, w.pid) }
-    end
-
-    def soft_kill_each_child(signal)
-      @children.each { |worker| worker.soft_kill(signal) }
-    end
-
     # returns an array of string names for the given listener array
     def listener_names(listeners = LISTENERS)
       listeners.map { |io| sock_name(io) }

data/lib/pitchfork/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Pitchfork
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
   module Const
     UNICORN_VERSION = '6.1.0'
   end

data/lib/pitchfork/worker.rb

@@ -139,6 +139,14 @@ module Pitchfork
       success
     end
 
+    def hard_kill(sig)
+      Process.kill(sig, pid)
+    end
+
+    def hard_timeout!
+      hard_kill(:KILL)
+    end
+
     # this only runs when the Rack app.call is not running
     # act like a listener
     def accept_nonblock(exception: nil) # :nodoc:

data/lib/pitchfork.rb

@@ -204,8 +204,13 @@ module Pitchfork
         # or the master to be PID 1.
         if middle_pid = FORK_LOCK.synchronize { Process.fork } # parent
           # We need to wait(2) so that the middle process doesn't end up a zombie.
-          Process.wait(middle_pid)
+          # The process only call fork again an exit so it should be pretty fast.
+          # However it might need to execute some `Process._fork` or `at_exit` callbacks,
+          # so it case it takes more than 5 seconds to exit, we kill it with SIGBUS
+          # to produce a crash report, as this is indicative of a nasty bug.
+          process_wait_with_timeout(middle_pid, 5, :BUS)
         else # first child
+          Process.setproctitle("<pitchfork fork_sibling>")
           clean_fork(&block) # detach into a grand child
           exit
         end
@@ -216,6 +221,18 @@ module Pitchfork
       nil # it's tricky to return the PID
     end
 
+    def process_wait_with_timeout(pid, timeout, timeout_signal = :KILL)
+      (timeout * 200).times do
+        status = Process.wait(pid, Process::WNOHANG)
+        return status if status
+        sleep 0.005 # 200 * 5ms => 1s
+      end
+
+      # The process didn't exit in the allotted time, so we kill it.
+      Process.kill(timeout_signal, pid)
+      Process.wait(pid)
+    end
+
     def time_now(int = false)
       Process.clock_gettime(Process::CLOCK_MONOTONIC, int ? :second : :float_second)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pitchfork
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-28 00:00:00.000000000 Z
+date: 2023-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: raindrops