pitchfork 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05372d35dd4784eb22e204b614b1c7add13fd0e298495543c5395e2f03b42073
-  data.tar.gz: 14f241efeb95774f6de6e9fc8926815eb7f10c25965a09a0700f77dae75dbb92
+  metadata.gz: 9c9d47a7cd3604f0807a41882322b4461edf2a990439e152dfa2d94bb731eff7
+  data.tar.gz: 1a38267df9cff0493452fa88c65cb4c9a502643aaf05c8e0b60544f93e9f6e4f
 SHA512:
-  metadata.gz: f26ebeeb3bc9f7533d25cc41b29a317fb8a80ce108a1859657adfa80b9eb8f88812cfe85dfecf0fbd8093b6e91f45c6ce51ab7b9a15864d96b9d0f5f77835786
-  data.tar.gz: 8d1f09bb24226f2c89b2db2d549c28508a9f2ea716a1612fcc3367d27445657b9d9da281e623af696dc803acd649847a724366306caf672dbc523a3e7495579b
+  metadata.gz: e8318fc2ae118a7e4a89f65e76e0634d306abd1838f0e4957f638bc870ddd202bc72c006fe3b93b6eec52f1765daea43b24b68eb7b8249b5b21b28a8b9abd51b
+  data.tar.gz: 5f23586cf49e29649496e15577ce7344c477b99878bfd6756f685f2a96ad40fc202d4f3cf97dad8183dd50b2489423c1fcfd48e20303d4eafae6b7f1db586e9b

data/.devcontainer/devcontainer.json

@@ -0,0 +1,28 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/ruby
+{
+	"name": "Ruby",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"build": {
+			// Path is relative to the devcontainer.json file.
+			"dockerfile": "../Dockerfile"
+	},
+	"features": {
+		"ghcr.io/devcontainers/features/github-cli:1": {}
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "bundle install"
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Unreleased
 
+# 0.8.0
+
+- Add an `after_monitor_ready` callback, called in the monitor process at end of boot.
+- Implement `Pitchfork.prevent_fork` for use in background threads that synchronize native locks with the GVL released.
+
 # 0.7.0
 
 - Set nicer `proctile` to better see the state of the process tree at a glance.

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.2
+FROM mcr.microsoft.com/devcontainers/ruby:1-3.2-bookworm
 RUN apt-get update -y && apt-get install -y ragel socat netcat-traditional smem apache2-utils
 WORKDIR /app
 CMD [ "bash" ]

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pitchfork (0.7.0)
+    pitchfork (0.8.0)
       rack (>= 2.0)
       raindrops (~> 0.7)
 
@@ -10,7 +10,7 @@ GEM
   specs:
     minitest (5.15.0)
     nio4r (2.5.9)
-    puma (6.3.0)
+    puma (6.3.1)
       nio4r (~> 2.0)
     rack (3.0.8)
     raindrops (0.20.1)

data/docs/CONFIGURATION.md

@@ -253,12 +253,23 @@ The default Logger will log its output to STDERR.
 Because pitchfork several callbacks around the lifecycle of workers.
 It is often necessary to use these callbacks to close inherited connection after fork.
 
-Note that when reforking is available, the `pitchfork` master process won't load your application
-at all. As such for hooks executed in the master, you may need to explicitly load the parts of your
+Note that when reforking is available, the `pitchfork` monitor process won't load your application
+at all. As such for hooks executed in the monitor, you may need to explicitly load the parts of your
 application that are used in hooks.
 
 `pitchfork` also don't attempt to rescue hook errors. Raising from a worker hook will crash the worker,
-and raising from a master hook will bring the whole cluster down.
+and raising from a monitor hook will bring the whole cluster down.
+
+### `after_monitor_ready`
+
+Called by the monitor process after it's done booting the application and
+spawning the original workers.
+
+```ruby
+after_monitor_ready do |server|
+  server.logger.info("Monitor pid=#{Process.pid} ready")
+end
+```
 
 ### `after_mold_fork`
 
@@ -338,7 +349,7 @@ By default the cleanup timeout is 2 seconds.
 
 ### `after_worker_hard_timeout`
 
-Called in the master process when a worker hard timeout is elapsed:
+Called in the monitor process when a worker hard timeout is elapsed:
 
 ```ruby
 after_worker_timeout do |server, worker|
@@ -353,7 +364,7 @@ soft timeout from working.
 
 ### `after_worker_exit`
 
-Called in the master process after a worker exits.
+Called in the monitor process after a worker exits.
 
 ```ruby
 after_worker_exit do |server, worker, status|

data/lib/pitchfork/configurator.rb

@@ -60,6 +60,9 @@ module Pitchfork
       :after_worker_ready => lambda { |server, worker|
         server.logger.info("worker=#{worker.nr} gen=#{worker.generation} ready")
       },
+      :after_monitor_ready => lambda { |server|
+        server.logger.info("Monitor pid=#{Process.pid} ready")
+      },
       :after_worker_timeout => nil,
       :after_worker_hard_timeout => nil,
       :after_request_complete => nil,
@@ -141,6 +144,10 @@ module Pitchfork
       set_hook(:after_worker_ready, block_given? ? block : args[0])
     end
 
+    def after_monitor_ready(*args, &block)
+      set_hook(:after_monitor_ready, block_given? ? block : args[0], 1)
+    end
+
     def after_worker_timeout(*args, &block)
       set_hook(:after_worker_timeout, block_given? ? block : args[0], 3)
     end

data/lib/pitchfork/http_server.rb

@@ -80,7 +80,7 @@ module Pitchfork
                   :orig_app, :config, :ready_pipe,
                   :default_middleware, :early_hints
     attr_writer   :after_worker_exit, :before_worker_exit, :after_worker_ready, :after_request_complete,
-                  :refork_condition, :after_worker_timeout, :after_worker_hard_timeout
+                  :refork_condition, :after_worker_timeout, :after_worker_hard_timeout, :after_monitor_ready
 
     attr_reader :logger
     include Pitchfork::SocketHelper
@@ -212,6 +212,8 @@ module Pitchfork
         wait_for_pending_workers
       end
 
+      @after_monitor_ready&.call(self)
+
       self
     end
 

data/lib/pitchfork/info.rb

@@ -29,13 +29,7 @@ module Pitchfork
         end
 
         ObjectSpace.each_object(IO) do |io|
-          closed = begin
-            io.closed?
-          rescue IOError
-            true
-          end
-
-          if !closed && io.autoclose? && !ignored_ios.include?(io)
+          if io_open?(io) && io_autoclosed?(io) && !ignored_ios.include?(io)
             if io.is_a?(TCPSocket)
               # If we inherited a TCP Socket, calling #close directly could send FIN or RST.
               # So we first reopen /dev/null to avoid that.
@@ -73,6 +67,20 @@ module Pitchfork
       def shutting_down?
         SharedMemory.shutting_down?
       end
+
+      private
+
+      def io_open?(io)
+        !io.closed?
+      rescue IOError
+        false
+      end
+
+      def io_autoclosed?(io)
+        io.autoclose?
+      rescue IOError
+        false
+      end
     end
   end
 end

data/lib/pitchfork/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Pitchfork
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
   module Const
     UNICORN_VERSION = '6.1.0'
   end

data/lib/pitchfork.rb

@@ -36,157 +36,190 @@ module Pitchfork
 
   # :stopdoc:
 
-  # This returns a lambda to pass in as the app, this does not "build" the
-  # app The returned lambda will be called when it is
-  # time to build the app.
-  def self.builder(ru, op)
-    # allow Configurator to parse cli switches embedded in the ru file
-    op = Pitchfork::Configurator::RACKUP.merge!(:file => ru, :optparse => op)
-    if ru =~ /\.ru$/ && !defined?(Rack::Builder)
-      abort "rack and Rack::Builder must be available for processing #{ru}"
+  FORK_LOCK = Monitor.new
+  @socket_type = :SOCK_SEQPACKET
+
+  class << self
+    # :startdoc:
+
+    # Prevent Pitchfork from forking new children for the duration of the block.
+    #
+    # If you have background threads calling code that synchronize native locks,
+    # while the GVL is released, forking while they are held could leak to
+    # corrupted children.
+    #
+    # One example of this is `getaddrinfo(3)`, so opening a connection from a
+    # background thread has a chance to produce stuck children.
+    #
+    # To avoid this you can wrap such code in `Pitchfork.prevent_fork`:
+    #
+    # def heartbeat_thread
+    #   @heartbeat_thread ||= Thread.new do
+    #     loop do
+    #       Pitchfork.prevent_fork do
+    #         heartbeat
+    #       end
+    #       sleep 10
+    #     end
+    #   end
+    # end
+    #
+    def prevent_fork(&block)
+      FORK_LOCK.synchronize(&block)
     end
 
-    # always called after config file parsing, may be called after forking
-    lambda do |_, server|
-      inner_app = case ru
-      when /\.ru$/
-        raw = File.read(ru)
-        raw.sub!(/^__END__\n.*/, '')
-        eval("Rack::Builder.new {(\n#{raw}\n)}.to_app", TOPLEVEL_BINDING, ru)
-      else
-        require ru
-        Object.const_get(File.basename(ru, '.rb').capitalize)
+    # :stopdoc:
+
+    # This returns a lambda to pass in as the app, this does not "build" the
+    # app The returned lambda will be called when it is
+    # time to build the app.
+    def builder(ru, op)
+      # allow Configurator to parse cli switches embedded in the ru file
+      op = Pitchfork::Configurator::RACKUP.merge!(:file => ru, :optparse => op)
+      if ru =~ /\.ru$/ && !defined?(Rack::Builder)
+        abort "rack and Rack::Builder must be available for processing #{ru}"
       end
 
-      Rack::Builder.new do
-        use(Rack::ContentLength)
-        use(Pitchfork::Chunked)
-        use(Rack::Lint) if ENV["RACK_ENV"] == "development"
-        use(Rack::TempfileReaper)
-        run inner_app
-      end.to_app
+      # always called after config file parsing, may be called after forking
+      lambda do |_, server|
+        inner_app = case ru
+        when /\.ru$/
+          raw = File.read(ru)
+          raw.sub!(/^__END__\n.*/, '')
+          eval("Rack::Builder.new {(\n#{raw}\n)}.to_app", TOPLEVEL_BINDING, ru)
+        else
+          require ru
+          Object.const_get(File.basename(ru, '.rb').capitalize)
+        end
+
+        Rack::Builder.new do
+          use(Rack::ContentLength)
+          use(Pitchfork::Chunked)
+          use(Rack::Lint) if ENV["RACK_ENV"] == "development"
+          use(Rack::TempfileReaper)
+          run inner_app
+        end.to_app
+      end
     end
-  end
 
-  # returns an array of strings representing TCP listen socket addresses
-  # and Unix domain socket paths.  This is useful for use with
-  # Raindrops::Middleware under Linux: https://yhbt.net/raindrops/
-  def self.listener_names
-    Pitchfork::HttpServer::LISTENERS.map do |io|
-      Pitchfork::SocketHelper.sock_name(io)
+    # returns an array of strings representing TCP listen socket addresses
+    # and Unix domain socket paths.  This is useful for use with
+    # Raindrops::Middleware under Linux: https://yhbt.net/raindrops/
+    def listener_names
+      Pitchfork::HttpServer::LISTENERS.map do |io|
+        Pitchfork::SocketHelper.sock_name(io)
+      end
     end
-  end
 
-  def self.log_error(logger, prefix, exc)
-    message = exc.message
-    message = message.dump if /[[:cntrl:]]/ =~ message
-    logger.error "#{prefix}: #{message} (#{exc.class})"
-    exc.backtrace.each { |line| logger.error(line) }
-  end
+    def log_error(logger, prefix, exc)
+      message = exc.message
+      message = message.dump if /[[:cntrl:]]/ =~ message
+      logger.error "#{prefix}: #{message} (#{exc.class})"
+      exc.backtrace.each { |line| logger.error(line) }
+    end
 
-  F_SETPIPE_SZ = 1031 if RUBY_PLATFORM =~ /linux/
-
-  def self.pipe # :nodoc:
-    IO.pipe.each do |io|
-      # shrink pipes to minimize impact on /proc/sys/fs/pipe-user-pages-soft
-      # limits.
-      if defined?(F_SETPIPE_SZ)
-        begin
-          io.fcntl(F_SETPIPE_SZ, Raindrops::PAGE_SIZE)
-        rescue Errno::EINVAL
-          # old kernel
-        rescue Errno::EPERM
-          # resizes fail if Linux is close to the pipe limit for the user
-          # or if the user does not have permissions to resize
+    F_SETPIPE_SZ = 1031 if RUBY_PLATFORM =~ /linux/
+
+    def pipe # :nodoc:
+      IO.pipe.each do |io|
+        # shrink pipes to minimize impact on /proc/sys/fs/pipe-user-pages-soft
+        # limits.
+        if defined?(F_SETPIPE_SZ)
+          begin
+            io.fcntl(F_SETPIPE_SZ, Raindrops::PAGE_SIZE)
+          rescue Errno::EINVAL
+            # old kernel
+          rescue Errno::EPERM
+            # resizes fail if Linux is close to the pipe limit for the user
+            # or if the user does not have permissions to resize
+          end
         end
       end
     end
-  end
 
-  @socket_type = :SOCK_SEQPACKET
-  def self.socketpair
-    pair = UNIXSocket.socketpair(@socket_type).map { |s| MessageSocket.new(s) }
-    pair[0].close_write
-    pair[1].close_read
-    pair
-  rescue Errno::EPROTONOSUPPORT
-    if @socket_type == :SOCK_SEQPACKET
-      # macOS and very old linuxes don't support SOCK_SEQPACKET (SCTP).
-      # In such case we can fallback to SOCK_STREAM (TCP)
-      warn("SEQPACKET (SCTP) isn't supported, falling back to STREAM")
-      @socket_type = :SOCK_STREAM
-      retry
-    else
-      raise
+    def socketpair
+      pair = UNIXSocket.socketpair(@socket_type).map { |s| MessageSocket.new(s) }
+      pair[0].close_write
+      pair[1].close_read
+      pair
+    rescue Errno::EPROTONOSUPPORT
+      if @socket_type == :SOCK_SEQPACKET
+        # macOS and very old linuxes don't support SOCK_SEQPACKET (SCTP).
+        # In such case we can fallback to SOCK_STREAM (TCP)
+        warn("SEQPACKET (SCTP) isn't supported, falling back to STREAM")
+        @socket_type = :SOCK_STREAM
+        retry
+      else
+        raise
+      end
     end
-  end
 
-  def self.clean_fork(setpgid: true, &block)
-    if pid = Process.fork
-      if setpgid
-        Process.setpgid(pid, pid) # Make into a group leader
+    def clean_fork(setpgid: true, &block)
+      if pid = FORK_LOCK.synchronize { Process.fork }
+        if setpgid
+          Process.setpgid(pid, pid) # Make into a group leader
+        end
+        return pid
       end
-      return pid
-    end
 
-    begin
-      # Pitchfork recursively refork the worker processes.
-      # Because of this we need to unwind the stack before resuming execution
-      # in the child, otherwise on each generation the available stack space would
-      # get smaller and smaller until it's basically 0.
-      #
-      # The very first version of this method used to call fork from a new
-      # thread, however this can cause issues with some native gems that rely on
-      # pthread_atfork(3) or pthread_mutex_lock(3), as the new main thread would
-      # now be different.
-      #
-      # A second version used to fork from a new fiber, but fibers have a much smaller
-      # stack space (https://bugs.ruby-lang.org/issues/3187), so it would break large applications.
-      #
-      # The latest version now use `throw` to unwind the stack after the fork, it however
-      # restrict it to be called only inside `handle_clean_fork`.
-      if Thread.current[:pitchfork_handle_clean_fork]
-        throw self, block
-      else
-        while block
-          block = catch(self) do
-            Thread.current[:pitchfork_handle_clean_fork] = true
-            block.call
-            nil
+      begin
+        # Pitchfork recursively refork the worker processes.
+        # Because of this we need to unwind the stack before resuming execution
+        # in the child, otherwise on each generation the available stack space would
+        # get smaller and smaller until it's basically 0.
+        #
+        # The very first version of this method used to call fork from a new
+        # thread, however this can cause issues with some native gems that rely on
+        # pthread_atfork(3) or pthread_mutex_lock(3), as the new main thread would
+        # now be different.
+        #
+        # A second version used to fork from a new fiber, but fibers have a much smaller
+        # stack space (https://bugs.ruby-lang.org/issues/3187), so it would break large applications.
+        #
+        # The latest version now use `throw` to unwind the stack after the fork, it however
+        # restrict it to be called only inside `handle_clean_fork`.
+        if Thread.current[:pitchfork_handle_clean_fork]
+          throw self, block
+        else
+          while block
+            block = catch(self) do
+              Thread.current[:pitchfork_handle_clean_fork] = true
+              block.call
+              nil
+            end
           end
         end
+      rescue
+        abort
+      else
+        exit
       end
-    rescue
-      abort
-    else
-      exit
     end
-  end
 
-  def self.fork_sibling(&block)
-    if REFORKING_AVAILABLE
-      # We double fork so that the new worker is re-attached back
-      # to the master.
-      # This requires either PR_SET_CHILD_SUBREAPER which is exclusive to Linux 3.4
-      # or the master to be PID 1.
-      if middle_pid = Process.fork # parent
-        # We need to wait(2) so that the middle process doesn't end up a zombie.
-        Process.wait(middle_pid)
-      else # first child
-        clean_fork(&block) # detach into a grand child
-        exit
+    def fork_sibling(&block)
+      if REFORKING_AVAILABLE
+        # We double fork so that the new worker is re-attached back
+        # to the master.
+        # This requires either PR_SET_CHILD_SUBREAPER which is exclusive to Linux 3.4
+        # or the master to be PID 1.
+        if middle_pid = FORK_LOCK.synchronize { Process.fork } # parent
+          # We need to wait(2) so that the middle process doesn't end up a zombie.
+          Process.wait(middle_pid)
+        else # first child
+          clean_fork(&block) # detach into a grand child
+          exit
+        end
+      else
+        clean_fork(&block)
       end
-    else
-      clean_fork(&block)
-    end
 
-    nil # it's tricky to return the PID
-  end
+      nil # it's tricky to return the PID
+    end
 
-  def self.time_now(int = false)
-    Process.clock_gettime(Process::CLOCK_MONOTONIC, int ? :second : :float_second)
+    def time_now(int = false)
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, int ? :second : :float_second)
+    end
   end
-  # :startdoc:
 end
 # :enddoc:
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pitchfork
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-18 00:00:00.000000000 Z
+date: 2023-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: raindrops
@@ -49,6 +49,7 @@ extensions:
 - ext/pitchfork_http/extconf.rb
 extra_rdoc_files: []
 files:
+- ".devcontainer/devcontainer.json"
 - ".git-blame-ignore-revs"
 - ".gitattributes"
 - ".github/workflows/ci.yml"