pitchfork 0.4.0 → 0.5.0

data/lib/pitchfork/chunked.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "rack"
-if Rack::VERSION.first >= 3
+if defined?(Rack::RELEASE) && Rack::RELEASE > "3"
   require "rack/constants"
   require "rack/utils"
 end

data/lib/pitchfork/configurator.rb

@@ -27,7 +27,9 @@ module Pitchfork
 
     # Default settings for Pitchfork
     DEFAULTS = {
-      :timeout => 20,
+      :soft_timeout => 20,
+      :cleanup_timeout => 2,
+      :timeout => 22,
       :logger => Logger.new($stderr),
       :worker_processes => 1,
       :after_worker_fork => lambda { |server, worker|
@@ -53,6 +55,8 @@ module Pitchfork
       :after_worker_ready => lambda { |server, worker|
         server.logger.info("worker=#{worker.nr} gen=#{worker.generation} ready")
       },
+      :after_worker_timeout => nil,
+      :after_request_complete => nil,
       :early_hints => false,
       :refork_condition => nil,
       :check_client_connection => false,
@@ -131,15 +135,22 @@ module Pitchfork
       set_hook(:after_worker_ready, block_given? ? block : args[0])
     end
 
+    def after_worker_timeout(*args, &block)
+      set_hook(:after_worker_timeout, block_given? ? block : args[0], 3)
+    end
+
     def after_worker_exit(*args, &block)
       set_hook(:after_worker_exit, block_given? ? block : args[0], 3)
     end
 
-    def timeout(seconds)
-      set_int(:timeout, seconds, 3)
-      # POSIX says 31 days is the smallest allowed maximum timeout for select()
-      max = 30 * 60 * 60 * 24
-      set[:timeout] = seconds > max ? max : seconds
+    def after_request_complete(*args, &block)
+      set_hook(:after_request_complete, block_given? ? block : args[0])
+    end
+
+    def timeout(seconds, cleanup: 2)
+      soft_timeout = set_int(:soft_timeout, seconds, 3)
+      cleanup_timeout = set_int(:cleanup_timeout, cleanup, 2)
+      set_int(:timeout, soft_timeout + cleanup_timeout, 5)
     end
 
     def worker_processes(nr)

data/lib/pitchfork/http_server.rb

@@ -1,6 +1,7 @@
 # -*- encoding: binary -*-
 require 'pitchfork/pitchfork_http'
 require 'pitchfork/flock'
+require 'pitchfork/soft_timeout'
 
 module Pitchfork
   # This is the process manager of Pitchfork. This manages worker
@@ -8,13 +9,75 @@ module Pitchfork
   # Listener sockets are started in the master process and shared with
   # forked worker children.
   class HttpServer
+    class TimeoutHandler
+      class Info
+        attr_reader :thread, :rack_env
+
+        def initialize(thread, rack_env)
+          @thread = thread
+          @rack_env = rack_env
+        end
+
+        def copy_thread_variables!
+          current_thread = Thread.current
+          @thread.keys.each do |key|
+            current_thread[key] = @thread[key]
+          end
+          @thread.thread_variables.each do |variable|
+            current_thread.thread_variable_set(variable, @thread.thread_variable_get(variable))
+          end
+        end
+      end
+
+      attr_writer :rack_env, :timeout_request # :nodoc:
+
+      def initialize(server, worker, callback) # :nodoc:
+        @server = server
+        @worker = worker
+        @callback = callback
+        @rack_env = nil
+        @timeout_request = nil
+      end
+
+      def inspect
+        "#<Pitchfork::HttpServer::TimeoutHandler##{object_id}>"
+      end
+
+      def call(original_thread) # :nodoc:
+        @server.logger.error("worker=#{@worker.nr} pid=#{@worker.pid} timed out, exiting")
+        if @callback
+          @callback.call(@server, @worker, Info.new(original_thread, @rack_env))
+        end
+      rescue => error
+        Pitchfork.log_error(@server.logger, "after_worker_timeout error", error)
+      ensure
+        exit
+      end
+
+      def finished # :nodoc:
+        @timeout_request.finished
+      end
+
+      def deadline
+        @timeout_request.deadline
+      end
+
+      def extend_deadline(extra_time)
+        extra_time = Integer(extra_time)
+        @worker.deadline += extra_time
+        @timeout_request.extend_deadline(extra_time)
+        self
+      end
+    end
+
     # :stopdoc:
-    attr_accessor :app, :timeout, :worker_processes,
+    attr_accessor :app, :timeout, :soft_timeout, :cleanup_timeout, :worker_processes,
                   :after_worker_fork, :after_mold_fork,
                   :listener_opts, :children,
                   :orig_app, :config, :ready_pipe,
                   :default_middleware, :early_hints
-    attr_writer   :after_worker_exit, :after_worker_ready, :refork_condition
+    attr_writer   :after_worker_exit, :after_worker_ready, :after_request_complete, :refork_condition,
+                  :after_worker_timeout
 
     attr_reader :logger
     include Pitchfork::SocketHelper
@@ -376,7 +439,7 @@ module Pitchfork
         if worker
           @after_worker_exit.call(self, worker, status)
         else
-          logger.error("reaped unknown subprocess #{status.inspect}")
+          logger.info("reaped unknown subprocess #{status.inspect}")
         end
       rescue Errno::ECHILD
         break
@@ -394,25 +457,30 @@ module Pitchfork
 
     # forcibly terminate all workers that haven't checked in in timeout seconds.  The timeout is implemented using an unlinked File
     def murder_lazy_workers
-      next_sleep = @timeout - 1
       now = Pitchfork.time_now(true)
+      next_sleep = @timeout - 1
+
       @children.workers.each do |worker|
-        tick = worker.tick
-        0 == tick and next # skip workers that haven't processed any clients
-        diff = now - tick
-        tmp = @timeout - diff
-        if tmp >= 0
-          next_sleep > tmp and next_sleep = tmp
+        deadline = worker.deadline
+        if 0 == deadline # worker is idle
           next
+        elsif deadline > now # worker still has time
+          time_left = deadline - now
+          if time_left < next_sleep
+            next_sleep = time_left
+          end
+          next
+        else # worker is out of time
+          next_sleep = 0
+          if worker.mold?
+            logger.error "mold pid=#{worker.pid} deadline=#{deadline} timed out, killing"
+          else
+            logger.error "worker=#{worker.nr} pid=#{worker.pid} deadline=#{deadline} timed out, killing"
+          end
+          kill_worker(:KILL, worker.pid) # take no prisoners for hard timeout violations
         end
-        next_sleep = 0
-        if worker.mold?
-          logger.error "mold pid=#{worker.pid} timeout (#{diff}s > #{@timeout}s), killing"
-        else
-          logger.error "worker=#{worker.nr} pid=#{worker.pid} timeout (#{diff}s > #{@timeout}s), killing"
-        end
-        kill_worker(:KILL, worker.pid) # take no prisoners for timeout violations
       end
+
       next_sleep <= 0 ? 1 : next_sleep
     end
 
@@ -577,9 +645,12 @@ module Pitchfork
 
     # once a client is accepted, it is processed in its entirety here
     # in 3 easy steps: read request, call app, write app response
-    def process_client(client)
+    def process_client(client, timeout_handler)
+      env = nil
       @request = Pitchfork::HttpParser.new
       env = @request.read(client)
+      timeout_handler.rack_env = env
+      env["pitchfork.timeout"] = timeout_handler
 
       if early_hints
         env["rack.early_hints"] = lambda do |headers|
@@ -616,6 +687,7 @@ module Pitchfork
       handle_error(client, e)
     ensure
       env["rack.after_reply"].each(&:call) if env
+      timeout_handler.finished
     end
 
     def nuke_listeners!(readers)
@@ -684,7 +756,7 @@ module Pitchfork
 
       while readers[0]
         begin
-          worker.tick = Pitchfork.time_now(true)
+          worker.update_deadline(@timeout)
           while sock = ready.shift
             # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
             # but that will return false
@@ -697,15 +769,16 @@ module Pitchfork
               when Message
                 worker.update(client)
               else
-                process_client(client)
+                process_client(client, prepare_timeout(worker))
+                @after_request_complete&.call(self, worker)
                 worker.increment_requests_count
               end
-              worker.tick = Pitchfork.time_now(true)
+              worker.update_deadline(@timeout)
             end
           end
 
-          # timeout so we can .tick and keep parent from SIGKILL-ing us
-          worker.tick = Pitchfork.time_now(true)
+          # timeout so we can update .deadline and keep parent from SIGKILL-ing us
+          worker.update_deadline(@timeout)
 
           if @refork_condition && !worker.outdated?
             if @refork_condition.met?(worker, logger)
@@ -754,7 +827,7 @@ module Pitchfork
 
       while readers[0]
         begin
-          mold.tick = Pitchfork.time_now(true)
+          mold.update_deadline(@timeout)
           while sock = ready.shift
             # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
             # but that will return false
@@ -774,7 +847,7 @@ module Pitchfork
           end
 
           # timeout so we can .tick and keep parent from SIGKILL-ing us
-          mold.tick = Pitchfork.time_now(true)
+          mold.update_deadline(@timeout)
           waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
         rescue => e
           Pitchfork.log_error(@logger, "mold loop error", e) if readers[0]
@@ -832,5 +905,11 @@ module Pitchfork
       listeners.each { |addr| listen(addr) }
       raise ArgumentError, "no listeners" if LISTENERS.empty?
     end
+
+    def prepare_timeout(worker)
+      handler = TimeoutHandler.new(self, worker, @after_worker_timeout)
+      handler.timeout_request = SoftTimeout.request(@soft_timeout, handler)
+      handler
+    end
   end
 end

data/lib/pitchfork/soft_timeout.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Pitchfork
+  # :stopdoc:
+  module SoftTimeout
+    extend self
+
+    CONDVAR = ConditionVariable.new
+    QUEUE = Queue.new
+    QUEUE_MUTEX = Mutex.new
+    TIMEOUT_THREAD_MUTEX = Mutex.new
+    @timeout_thread = nil
+    private_constant :CONDVAR, :QUEUE, :QUEUE_MUTEX, :TIMEOUT_THREAD_MUTEX
+
+    class Request
+      attr_reader :deadline, :thread
+
+      def initialize(thread, timeout, block)
+        @thread = thread
+        @deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        @block = block
+
+        @mutex = Mutex.new
+        @done = false # protected by @mutex
+      end
+
+      def extend_deadline(timeout)
+        @deadline += timeout
+        QUEUE_MUTEX.synchronize do
+          CONDVAR.signal
+        end
+        self
+      end
+
+      def done?
+        @mutex.synchronize do
+          @done
+        end
+      end
+
+      def expired?(now)
+        now >= @deadline
+      end
+
+      def interrupt
+        @mutex.synchronize do
+          unless @done
+            begin
+              @block.call(@thread)
+            ensure
+              @done = true
+            end
+          end
+        end
+      end
+
+      def finished
+        @mutex.synchronize do
+          @done = true
+        end
+      end
+    end
+
+    def request(sec, callback)
+      ensure_timeout_thread_created
+      request = Request.new(Thread.current, sec, callback)
+      QUEUE_MUTEX.synchronize do
+        QUEUE << request
+        CONDVAR.signal
+      end
+      request
+    end
+
+    private
+
+    def create_timeout_thread
+      watcher = Thread.new do
+        requests = []
+        while true
+          until QUEUE.empty? and !requests.empty? # wait to have at least one request
+            req = QUEUE.pop
+            requests << req unless req.done?
+          end
+          closest_deadline = requests.min_by(&:deadline).deadline
+
+          now = 0.0
+          QUEUE_MUTEX.synchronize do
+            while (now = Process.clock_gettime(Process::CLOCK_MONOTONIC)) < closest_deadline and QUEUE.empty?
+              CONDVAR.wait(QUEUE_MUTEX, closest_deadline - now)
+            end
+          end
+
+          requests.each do |req|
+            req.interrupt if req.expired?(now)
+          end
+          requests.reject!(&:done?)
+        end
+      end
+      watcher.name = "Pitchfork::Timeout"
+      watcher
+    end
+
+    def ensure_timeout_thread_created
+      unless @timeout_thread and @timeout_thread.alive?
+        TIMEOUT_THREAD_MUTEX.synchronize do
+          unless @timeout_thread and @timeout_thread.alive?
+            @timeout_thread = create_timeout_thread
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pitchfork/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Pitchfork
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
   module Const
     UNICORN_VERSION = '6.1.0'
   end

data/lib/pitchfork/worker.rb

@@ -93,7 +93,7 @@ module Pitchfork
       @mold = true
       @nr = nil
       @drop_offset = 0
-      @tick_drop = MOLD_DROP
+      @deadline_drop = MOLD_DROP
       self
     end
 
@@ -167,21 +167,25 @@ module Pitchfork
       super || (!@nr.nil? && @nr == other)
     end
 
+    def update_deadline(timeout)
+      self.deadline = Pitchfork.time_now(true) + timeout
+    end
+
     # called in the worker process
-    def tick=(value) # :nodoc:
+    def deadline=(value) # :nodoc:
       if mold?
         MOLD_DROP[0] = value
       else
-        @tick_drop[@drop_offset] = value
+        @deadline_drop[@drop_offset] = value
       end
     end
 
     # called in the master process
-    def tick # :nodoc:
+    def deadline # :nodoc:
       if mold?
         MOLD_DROP[0]
       else
-        @tick_drop[@drop_offset]
+        @deadline_drop[@drop_offset]
       end
     end
 
@@ -221,7 +225,7 @@ module Pitchfork
         else
           success = true
         end
-      rescue Errno::EPIPE, Errno::ECONNRESET
+      rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNREFUSED
         # worker will be reaped soon
       end
       success
@@ -248,8 +252,8 @@ module Pitchfork
     def build_raindrops(drop_nr)
       drop_index = drop_nr / PER_DROP
       @drop_offset = drop_nr % PER_DROP
-      @tick_drop = TICK_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
-      @tick_drop[@drop_offset] = 0
+      @deadline_drop = TICK_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
+      @deadline_drop[@drop_offset] = 0
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pitchfork
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-13 00:00:00.000000000 Z
+date: 2023-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: raindrops
@@ -108,6 +108,7 @@ files:
 - lib/pitchfork/refork_condition.rb
 - lib/pitchfork/select_waiter.rb
 - lib/pitchfork/socket_helper.rb
+- lib/pitchfork/soft_timeout.rb
 - lib/pitchfork/stream_input.rb
 - lib/pitchfork/tee_input.rb
 - lib/pitchfork/tmpio.rb