gritz-core 0.2.0 → 0.3.0

data/lib/gritz/worker/runner.rb

@@ -7,27 +7,37 @@ module Gritz
     # Owns transport resources and heartbeats in one serving process.
     # @api private
     class Runner
-      def initialize(index:, config:, logger:, status_io: nil)
+      def initialize(index:, config:, logger:, status_io: nil, owner_channel: nil)
         @index = index
         @config = config
         @logger = logger
-        @status = Supervisor::StatusChannel.new(status_io) if status_io
+        @owner_channel = owner_channel
+        @status = owner_channel || (Supervisor::StatusChannel.new(status_io) if status_io)
+        @recorder = Metrics::Recorder.new
+        @metrics = Metrics::Aggregator.new
+        @metric_seq = 0
+        @born_at = monotonic
       end
 
       def run
         @exit_status = 0
         begin
-          @signals = Supervisor::SignalQueue.new(signals: %w[TERM INT QUIT HUP])
+          @signals = Supervisor::SignalQueue.new(signals: %w[TERM INT QUIT HUP USR1 USR2])
           report("booting")
           require "gritz/native"
           @boot_started = true
           @config.preload! unless @config.preload_app?
           @config.run_hooks(:on_worker_boot, @index)
           router = Router.new(controllers: @config.controllers, strict: @config.strict_routes, logger: @logger)
-          dispatcher = Dispatcher.new(router:, middleware: @config.middleware, logger: @logger)
+          dispatcher = Dispatcher.new(router:, middleware: @config.middleware, logger: @logger, metrics: @recorder,
+                                      worker: @index, log_format: @config.log_format, log_redact: @config.log_redact)
           @adapter = Transport::Native.new(config: @config, dispatcher:, logger: @logger)
           @port = @adapter.bind(@config.bind)
           @adapter.start
+          unless @status
+            @admin = Supervisor::AdminServer.new(bind: @config.admin_bind, status: -> { snapshot }, ready: -> { ready? },
+                                                 metrics: -> { @metrics.render(workers: [@last_status].compact) }, logger: @logger)
+          end
           @logger.info("Gritz #{Core::VERSION} listening on #{@config.bind.sub(/:\d+\z/, ":#{@port}")}")
           report("ready")
           serve
@@ -50,35 +60,99 @@ module Gritz
           @signals.drain.each do |name|
             case name
             when "HUP" then @logger.reopen
+            when "USR1"
+              if @config.workers.zero?
+                @logger.warn("USR1 requires workers > 0; use USR2 to reload a single-process server")
+              else
+                drain
+              end
+            when "USR2"
+              if @owner_channel
+                @pending_reexec = true
+              else
+                @logger.warn("USR2 requires the gritz launcher")
+              end
             when "QUIT"
-              report("draining")
+              drain
               @adapter.kill
               @transport_stopped = true
               return 0
             when "TERM", "INT"
-              report("draining")
-              @adapter.stop(deadline: Time.now + @config.shutdown_timeout)
-              @transport_stopped = true
-              return 0
+              drain
+              # Supervised children have already waited in the master's drain phase.
+              @drain_until ||= monotonic + (@config.workers.zero? ? @config.drain_delay : 0)
             end
           end
           now = monotonic
+          if @drain_until && now >= @drain_until
+            @stop_deadline = now + @config.shutdown_timeout
+            @adapter.stop(deadline: Time.now + @config.shutdown_timeout)
+            @transport_stopped = true
+            return 0
+          end
+          @status&.flush
+          if @pending_reexec && @owner_channel.write(type: "reexec", pid: Process.pid)
+            @pending_reexec = false
+          end
+          publish_metrics
+          @admin&.poll
           if now >= next_status
-            report("ready")
+            report(@draining ? "draining" : "ready")
             next_status = now + @config.status_interval
           end
-          @signals.io.wait_readable([next_status - monotonic, 0].max)
+          next_event = [next_status, @drain_until].compact.min
+          IO.select([@signals.io, *@admin&.ios.to_a], nil, nil, (next_event - monotonic).clamp(0, 0.05))
         end
       end
 
       def report(state, include_stats: true, **extra)
-        return unless @status
-
         stats = include_stats && @adapter ? @adapter.stats : {}
+        @recorder.observe_rejected(stats.fetch(:rejected_total, 0)) if include_stats && @adapter
+        evaluate_health if state == "ready"
         row = { inflight: 0, capacity: @config.threads, requests_total: 0, oldest_inflight_age: 0 }.merge(stats)
         row.merge!(pid: Process.pid, index: @index, state:, ts: monotonic, port: @port,
-                   busy_threads: stats.fetch(:busy_threads, stats.fetch(:busy, 0)))
-        @status.write(row.merge(extra))
+                   busy_threads: stats.fetch(:busy_threads, stats.fetch(:busy, 0)), healthy: state == "ready" && @healthy,
+                   checks: @checks || {}, worker_started_at: @born_at)
+        @last_status = row.merge(extra)
+        @status&.write(@owner_channel ? snapshot : @last_status)
+      end
+
+      def snapshot
+        { type: "status", pid: Process.pid, state: @draining ? "draining" : "running", desired: 1, workers: [@last_status].compact }
+      end
+
+      def ready? = !@draining && @last_status && @last_status[:state] == "ready" && @last_status[:healthy]
+
+      def evaluate_health
+        @checks = @config.health_checks.to_h do |name, callback|
+          [name, callback.call ? true : false]
+        rescue StandardError => e
+          @logger.warn("Health check #{name}: #{e.message}")
+          [name, false]
+        end
+        @healthy = @adapter.update_health(ready: true, checks: @checks)
+      end
+
+      def drain
+        return if @draining
+
+        @draining = true
+        @adapter.drain!
+        report("draining")
+      end
+
+      def publish_metrics
+        @pending_delta ||= @recorder.take_delta
+        return true unless @pending_delta
+
+        packet = { type: "metrics", pid: Process.pid, worker_pid: Process.pid, worker_started_at: @born_at,
+                   seq: @metric_seq + 1, delta: @pending_delta }
+        accepted = @status ? @status.write(packet) : @metrics.apply(self, packet)
+        if accepted
+          @metric_seq += 1
+          @pending_delta = nil
+        end
+        accepted
       end
 
       def fail_worker(error)
@@ -90,10 +164,28 @@ module Gritz
       def finish
         cleanup { @adapter&.kill unless @transport_stopped }
         cleanup { @config.run_hooks(:on_worker_shutdown, @index) if @boot_started }
-        report("stopped", include_stats: false, exit_status: @exit_status)
+        cleanup { @recorder.observe_rejected(@adapter.stats.fetch(:rejected_total, 0)) if @adapter }
+        deadline = @stop_deadline || (monotonic + @config.shutdown_timeout)
+        loop do
+          @status&.flush
+          accepted = publish_metrics
+          @pending_delta ||= @recorder.take_delta if accepted
+          break if accepted && !@pending_delta && (@status.nil? || @status.flush)
+          break if monotonic >= deadline || @status&.closed?
+
+          @status&.io&.wait_writable((deadline - monotonic).clamp(0, 0.01))
+        end
+        until report("stopped", include_stats: false, exit_status: @exit_status) != false
+          break if monotonic >= deadline || @status&.closed?
+
+          @status.flush
+          @status.io.wait_writable((deadline - monotonic).clamp(0, 0.01)) unless @status.closed?
+        end
+        @status.io.wait_writable((deadline - monotonic).clamp(0, 0.01)) until !@status || @status.flush || @status.closed? || monotonic >= deadline
       ensure
         begin
           @signals&.close
+          @admin&.close
         ensure
           @status&.close
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gritz-core
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Yudai Takada
@@ -9,6 +9,26 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -113,12 +133,17 @@ files:
 - lib/gritz/errors.rb
 - lib/gritz/fork_guard.rb
 - lib/gritz/method_descriptor.rb
+- lib/gritz/metrics/aggregator.rb
+- lib/gritz/metrics/recorder.rb
 - lib/gritz/middleware/context.rb
 - lib/gritz/middleware/exception_mapper.rb
 - lib/gritz/middleware/logging.rb
+- lib/gritz/middleware/metrics.rb
 - lib/gritz/middleware/request_id.rb
 - lib/gritz/middleware/stack.rb
 - lib/gritz/router.rb
+- lib/gritz/supervisor/admin_server.rb
+- lib/gritz/supervisor/launcher.rb
 - lib/gritz/supervisor/master.rb
 - lib/gritz/supervisor/signal_queue.rb
 - lib/gritz/supervisor/status_channel.rb