hyperion-rb 2.16.3 → 2.16.4

data/lib/hyperion/response_writer.rb

@@ -103,6 +103,47 @@ module Hyperion
     private
 
     def write_buffered(io, status, headers, body, keep_alive:)
+      if c_path_eligible?(io)
+        date_str = cached_date
+
+        # Plan #2 — io_uring hotpath write-side: when the current
+        # accept fiber owns a HotpathRing AND the C ext exposes the
+        # via-ring entrypoint, route the response through a send SQE
+        # instead of issuing sendmsg(2) directly. The hotpath ring is
+        # set as a fiber-local by Server#run_accept_fiber_io_uring_hotpath
+        # (Task 2.3.3) and inherited by the dispatch fiber. Ring is
+        # nil for non-hotpath connections (TLS / accept-only ring /
+        # epoll path) — we fall through to direct-syscall write.
+        ring = Fiber[:hyperion_hotpath_ring]
+        if ring && !ring.closed? &&
+           ::Hyperion::Http::ResponseWriter.respond_to?(:c_write_buffered_via_ring)
+          bytes_out = ::Hyperion::Http::ResponseWriter.c_write_buffered_via_ring(
+            io, status, headers, Array(body), keep_alive, date_str, ring.ptr
+          )
+          Hyperion.metrics.increment(:bytes_written, bytes_out)
+          Hyperion.metrics.increment(:hotpath_writes_via_ring)
+          body.close if body.respond_to?(:close)
+          return
+        end
+
+        bytes_out = ::Hyperion::Http::ResponseWriter.c_write_buffered(
+          io, status, headers, Array(body), keep_alive, date_str
+        )
+        if bytes_out == ::Hyperion::Http::ResponseWriter::WOULDBLOCK
+          # EAGAIN on the C path — fall back to the Ruby writer, which
+          # yields the fiber under Async / blocks the thread under
+          # threadpool correctly.
+          return write_buffered_ruby(io, status, headers, body, keep_alive: keep_alive)
+        end
+        Hyperion.metrics.increment(:bytes_written, bytes_out)
+        body.close if body.respond_to?(:close)
+        return
+      end
+
+      write_buffered_ruby(io, status, headers, body, keep_alive: keep_alive)
+    end
+
+    def write_buffered_ruby(io, status, headers, body, keep_alive:)
       # Phase 1 buffers the full body so Content-Length is exact.
       # Phase 2 introduces chunked transfer-encoding for streaming bodies;
       # Phase 5 batches via IO::Buffer to avoid this intermediate String.
@@ -386,6 +427,55 @@ module Hyperion
       false
     end
 
+    # Plan #1 (perf roadmap) — predicate for the C-side direct-syscall
+    # write path. True when:
+    #   (a) The Hyperion::Http::ResponseWriter C ext loaded.
+    #   (b) `io` exposes a real kernel fd (real_fd_io? handles the
+    #       SSLSocket / StringIO / IO-like-but-no-fileno cases).
+    #   (c) The class-level operator switch hasn't been flipped off.
+    #
+    # Operators flip the switch off via:
+    #   Hyperion::Http::ResponseWriter.c_writer_available = false
+    # (mirrors the Hyperion::ResponseWriter.page_cache_available pattern).
+    def c_path_eligible?(io)
+      return false unless defined?(::Hyperion::Http::ResponseWriter)
+      return false unless ::Hyperion::Http::ResponseWriter.c_writer_available?
+      return false unless real_fd_io?(io)
+
+      true
+    end
+
+    def write_chunked(io, status, headers, body, keep_alive:)
+      if c_path_eligible?(io)
+        date_str = cached_date
+        begin
+          bytes_out = ::Hyperion::Http::ResponseWriter.c_write_chunked(
+            io, status, headers, body, keep_alive, date_str
+          )
+        rescue Errno::EAGAIN
+          # Mid-body backpressure on the chunked path leaves the wire
+          # in a half-written state; we cannot recover by falling back
+          # to the Ruby writer because the chunked head has already
+          # been emitted. Re-raise so Connection#serve closes the
+          # connection cleanly. This is the documented contract from
+          # response_writer.c hyp_chunked_drain.
+          raise
+        end
+        if bytes_out == ::Hyperion::Http::ResponseWriter::WOULDBLOCK
+          # Pre-body WOULDBLOCK — only the head failed to ship; nothing
+          # is on the wire yet. Fall back to the Ruby chunked writer,
+          # which yields under Async / blocks under threadpool correctly.
+          return write_chunked_ruby(io, status, headers, body, keep_alive: keep_alive)
+        end
+        Hyperion.metrics.increment(:bytes_written, bytes_out)
+        Hyperion.metrics.increment(:chunked_responses)
+        body.close if body.respond_to?(:close)
+        return
+      end
+
+      write_chunked_ruby(io, status, headers, body, keep_alive: keep_alive)
+    end
+
     # Phase 5 — streaming chunked writer with per-response coalescing.
     #
     # Wire format per RFC 7230 §4.1:
@@ -405,7 +495,7 @@ module Hyperion
     #     past per-event coalescing latency.
     #   * body.close (or end-of-each) drains the buffer and appends the
     #     0\r\n\r\n terminator in a single syscall (atomic w.r.t. the wire).
-    def write_chunked(io, status, headers, body, keep_alive:)
+    def write_chunked_ruby(io, status, headers, body, keep_alive:)
       reason = REASONS[status] || 'Unknown'
       date_str = cached_date
       head = build_head_chunked(status, reason, headers, keep_alive, date_str)

data/lib/hyperion/server.rb

@@ -195,6 +195,7 @@ module Hyperion
                    tls_session_cache_size: TLS::DEFAULT_SESSION_CACHE_SIZE,
                    tls_ktls: :auto,
                    io_uring: :off,
+                   io_uring_hotpath: :off,
                    max_in_flight_per_conn: nil,
                    tls_handshake_rate_limit: :unlimited,
                    route_table: nil,
@@ -243,6 +244,17 @@ module Hyperion
       @io_uring_policy          = io_uring
       @io_uring_active          = io_uring != :off && Hyperion::IOUring.resolve_policy!(io_uring)
       log_io_uring_state_once
+      # Plan #2 — hotpath gate. Independent of @io_uring_active: the
+      # hotpath owns multishot accept + multishot recv + send SQEs on a
+      # single unified ring, while the accept-only path uses a simpler
+      # ring that only drives accept SQEs. The two paths are mutually
+      # exclusive at runtime — HotpathRing takes priority when active.
+      # Workers don't share hotpath rings across fork; each child opens
+      # its own ring lazily on first use inside `run_accept_fiber`.
+      @io_uring_hotpath_policy  = io_uring_hotpath
+      @io_uring_hotpath_active  = io_uring_hotpath != :off &&
+                                   Hyperion::IOUring.resolve_hotpath_policy!(io_uring_hotpath)
+      log_io_uring_hotpath_state_once
       # 2.3-B: per-conn fairness cap (validated/finalized upstream by
       # `Config#finalize!`; constructor accepts the resolved value, not
       # a sentinel). nil = no cap (default). The cap propagates to
@@ -880,8 +892,15 @@ module Hyperion
     # epoll branch — io_uring accept is wired only for the plain TCP
     # listener; the SSL handshake still wants the userspace
     # `accept` + `SSL_accept` dance.
+    #
+    # Plan #2: when `io_uring_hotpath: :auto/:on` resolves to active, the
+    # hotpath ring takes priority — it owns multishot accept + multishot recv
+    # + send SQEs. The accept-only ring and the epoll path are mutually
+    # exclusive with it. TLS still uses the epoll branch regardless.
     def run_accept_fiber(task)
-      if @io_uring_active && !@tls
+      if @io_uring_hotpath_active && !@tls
+        run_accept_fiber_io_uring_hotpath(task)
+      elsif @io_uring_active && !@tls
         run_accept_fiber_io_uring(task)
       else
         run_accept_fiber_epoll(task)
@@ -905,7 +924,7 @@ module Hyperion
     # working off a `::Socket` object identical to what
     # `accept_nonblock` would have returned.
     def run_accept_fiber_io_uring(task)
-      ring = Fiber.current[:hyperion_io_uring] ||= Hyperion::IOUring::Ring.new(queue_depth: 256)
+      ring = Fiber[:hyperion_io_uring] ||= Hyperion::IOUring::Ring.new(queue_depth: 256)
       listener_fd = listening_io.fileno
       until @stopped
         client_fd = ring.accept(listener_fd)
@@ -925,13 +944,168 @@ module Hyperion
       end
       run_accept_fiber_epoll(task)
     ensure
-      ring = Fiber.current[:hyperion_io_uring]
+      ring = Fiber[:hyperion_io_uring]
       if ring && !ring.closed?
         ring.close
-        Fiber.current[:hyperion_io_uring] = nil
+        Fiber[:hyperion_io_uring] = nil
       end
     end
 
+    # Plan #2 — io_uring hotpath accept loop. Opens a per-fiber
+    # HotpathRing on first use (multishot accept + multishot recv +
+    # send SQEs on one unified ring). For this task the loop only
+    # submits the multishot accept SQE and drains accept completions;
+    # the per-connection recv wiring is added in Task 2.3.4.
+    #
+    # On failure it closes the hotpath ring and falls back to the epoll
+    # path, matching the accept-only ring's fallback contract.
+    def run_accept_fiber_io_uring_hotpath(task)
+      ring = Fiber[:hyperion_hotpath_ring] ||=
+               Hyperion::IOUring::HotpathRing.new
+      listener_fd = listening_io.fileno
+      ring.submit_accept_multishot(listener_fd)
+      # Plan #2 Task 2.3.4 — per-connection state map.
+      # Maps client_fd (Integer) → Connection.  Populated on OP_ACCEPT
+      # and cleared on OP_RECV result <= 0 (EOF / error).  Single-fiber
+      # — no mutex needed.
+      hotpath_connections = {}
+      until @stopped
+        ring.each_completion(min_complete: 1, timeout_ms: 100) do |c|
+          case c[:op_kind]
+          when Hyperion::IOUring::HotpathRing::OP_ACCEPT
+            next if c[:result].negative?
+
+            client_fd = c[:result].to_i
+            socket = ::Socket.for_fd(client_fd)
+            socket.autoclose = true
+            apply_timeout(socket)
+            # Build a Connection for the accepted fd. io_uring_owned: true
+            # arms the guard in read_chunk (should never be called for
+            # these connections) and signals close_for_eof callers.
+            conn = Connection.new(runtime: @explicit_runtime ? @runtime : nil,
+                                  max_in_flight_per_conn: @max_in_flight_per_conn,
+                                  route_table: @route_table,
+                                  io_uring_owned: true,
+                                  app: @app)
+            conn.instance_variable_set(:@socket, socket)
+            # Server has no @metrics ivar — sibling accept paths bump
+            # these counters via runtime_metrics (record_dispatch et al.)
+            # or via Connection#serve, which we bypass here. Using @metrics
+            # here raised NoMethodError on the first ACCEPT completion and
+            # took down the --async-io + hotpath=on + 1w boot before
+            # wait_for_bind could land its first probe (row-19 BOOT-FAIL).
+            runtime_metrics.increment(:connections_accepted)
+            runtime_metrics.increment(:connections_active)
+            hotpath_connections[client_fd] = conn
+            # Post the first multishot-recv SQE for this fd.  From here
+            # on, the kernel delivers recv CQEs for every batch of bytes
+            # that arrives on this socket until EOF or cancel.
+            ring.submit_recv_multishot(client_fd)
+
+          when Hyperion::IOUring::HotpathRing::OP_RECV
+            fd     = c[:fd]
+            result = c[:result]
+            buf_id = c[:buf_id]
+
+            conn = hotpath_connections[fd]
+            next unless conn
+
+            if result <= 0
+              # 0 = peer EOF; negative = error.  Clean up and discard.
+              hotpath_connections.delete(fd)
+              conn.close_for_eof
+              begin
+                conn.socket&.close unless conn.socket&.closed?
+              rescue StandardError
+                nil
+              end
+            else
+              # Copy `result` bytes from the kernel buffer slot into a
+              # Ruby String (one allocation), then release the slot so
+              # the kernel can reuse it for the next recv.
+              bytes = ring.copy_buffer(buf_id, result)
+              ring.release_buffer(buf_id)
+              conn.feed_read_bytes(bytes)
+            end
+          end
+        end
+
+        # Plan #2 Task 2.5.2 — per-worker fallback-engaged detection.
+        # After each completion drain, check whether the ring went
+        # unhealthy (sustained SQE submit failures / repeated EBADR
+        # from the Rust side set a dirty flag). When detected:
+        #   1. Increment the observable metric so operators can alert.
+        #   2. Emit a single warn-level log line (actionable, not spammy).
+        #   3. Flip @io_uring_hotpath_active to false so subsequent
+        #      accept-fiber dispatch (run_accept_fiber's top-level branch)
+        #      uses the epoll path for newly-spawned accept fibers on
+        #      restart — the current fiber exits the loop and falls
+        #      through to run_accept_fiber_epoll below.
+        unless ring.healthy?
+          runtime_metrics.increment(:io_uring_hotpath_fallback_engaged)
+          runtime_logger.warn do
+            { message: 'io_uring hotpath ring unhealthy; engaging accept4 fallback per-worker',
+              worker_pid: Process.pid }
+          end
+          @io_uring_hotpath_active = false
+          begin
+            ring.close
+          rescue StandardError
+            nil
+          end
+          Fiber[:hyperion_hotpath_ring] = nil
+          break
+        end
+      end
+      # If we broke out due to an unhealthy ring (not a clean stop), fall
+      # through to the epoll path so existing + new connections keep being
+      # served. @stopped is still false in that case — the server is alive.
+      run_accept_fiber_epoll(task) unless @stopped
+    rescue IOError, Errno::EBADF
+      @stopped = true
+    rescue Hyperion::IOUring::Unsupported => e
+      runtime_logger.warn do
+        { message: 'io_uring hotpath unsupported at fiber open; falling back to epoll',
+          error: e.message }
+      end
+      close_hotpath_ring_for_fallback
+      run_accept_fiber_epoll(task)
+    rescue StandardError => e
+      runtime_logger.warn do
+        { message: 'io_uring hotpath accept fiber error; falling back to epoll',
+          error: e.message, error_class: e.class.name }
+      end
+      close_hotpath_ring_for_fallback
+      run_accept_fiber_epoll(task)
+    ensure
+      close_hotpath_ring_for_fallback
+    end
+
+    # Cancel the multishot-accept SQE armed on @listener_fd by closing
+    # the hotpath ring before the epoll fallback takes over. Without this,
+    # `accept_or_nil` in run_accept_fiber_epoll competes with a still-armed
+    # kernel-side multishot-accept consumer and inbound connections can be
+    # delivered to a dead CQ. Idempotent.
+    def close_hotpath_ring_for_fallback
+      ring = Fiber[:hyperion_hotpath_ring]
+      return unless ring && !ring.closed?
+
+      ring.close
+      Fiber[:hyperion_hotpath_ring] = nil
+    rescue StandardError
+      nil
+    end
+    private :close_hotpath_ring_for_fallback
+
+    # Plan #2 — test seam: returns the active HotpathRing on the current
+    # accept fiber, or nil if none. Used by io_uring_hotpath_fallback_engaged_spec
+    # to inject force_unhealthy! without exposing the ring through the
+    # public Server surface.
+    def hotpath_ring_for_test
+      Fiber[:hyperion_hotpath_ring]
+    end
+    private :hotpath_ring_for_test
+
     # Boot-time log line per worker capturing the resolved io_uring
     # state. Mirrors the `log_ktls_state_once` pattern from 2.2.0.
     # Single-shot via the class-level ivar so multi-worker boots
@@ -953,6 +1127,28 @@ module Hyperion
       nil
     end
 
+    # Plan #2 — boot-time log for the hotpath gate. Single-shot via the
+    # class-level ivar so multi-worker boots don't fan into N identical
+    # lines. Mirrors the log_io_uring_state_once pattern.
+    def log_io_uring_hotpath_state_once
+      return if Hyperion::Server.instance_variable_get(:@io_uring_hotpath_state_logged)
+      return if @io_uring_hotpath_policy == :off
+
+      Hyperion::Server.instance_variable_set(:@io_uring_hotpath_state_logged, true)
+      runtime_logger.info do
+        {
+          message: 'io_uring hotpath state',
+          policy: @io_uring_hotpath_policy,
+          active: @io_uring_hotpath_active,
+          kernel_ok: Hyperion::IOUring.kernel_supports_io_uring?,
+          hotpath_supported: Hyperion::IOUring.respond_to?(:hotpath_supported?) &&
+                              Hyperion::IOUring.hotpath_supported?
+        }
+      end
+    rescue StandardError
+      nil
+    end
+
     def dispatch(socket)
       alpn = socket.is_a?(::OpenSSL::SSL::SSLSocket) ? socket.alpn_protocol : nil
       mode = DispatchMode.resolve(tls: !@tls.nil?, async_io: @async_io,

data/lib/hyperion/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hyperion
-  VERSION = '2.16.3'
+  VERSION = '2.16.4'
 end

data/lib/hyperion.rb

@@ -263,6 +263,7 @@ require_relative 'hyperion/parser'
 require_relative 'hyperion/c_parser'
 require_relative 'hyperion/http/sendfile'
 require_relative 'hyperion/http/page_cache'
+require_relative 'hyperion/http/response_writer'
 require_relative 'hyperion/static_preload'
 require_relative 'hyperion/adapter/rack'
 require_relative 'hyperion/lint_wrapper_pool'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hyperion-rb
 version: !ruby/object:Gem::Version
-  version: 2.16.3
+  version: 2.16.4
 platform: ruby
 authors:
 - Andrey Lobanov
@@ -173,11 +173,15 @@ files:
 - ext/hyperion_http/page_cache.c
 - ext/hyperion_http/page_cache_internal.h
 - ext/hyperion_http/parser.c
+- ext/hyperion_http/response_writer.c
+- ext/hyperion_http/response_writer.h
 - ext/hyperion_http/sendfile.c
 - ext/hyperion_http/websocket.c
 - ext/hyperion_io_uring/Cargo.lock
 - ext/hyperion_io_uring/Cargo.toml
 - ext/hyperion_io_uring/extconf.rb
+- ext/hyperion_io_uring/src/buffer_ring.rs
+- ext/hyperion_io_uring/src/hotpath.rs
 - ext/hyperion_io_uring/src/lib.rs
 - lib/hyperion-rb.rb
 - lib/hyperion.rb
@@ -194,6 +198,7 @@ files:
 - lib/hyperion/h2_admission.rb
 - lib/hyperion/h2_codec.rb
 - lib/hyperion/http/page_cache.rb
+- lib/hyperion/http/response_writer.rb
 - lib/hyperion/http/sendfile.rb
 - lib/hyperion/http2/native_hpack_adapter.rb
 - lib/hyperion/http2_handler.rb