phronomy 0.13.0 → 0.15.0

data/lib/phronomy/engine/event_loop.rb

@@ -1,121 +1,61 @@
 # frozen_string_literal: true
 
 module Phronomy
-  # Singleton event loop that manages all FSMSession instances.
-  #
-  # A single background thread reads from a global {Phronomy::Concurrency::AsyncQueue} and
-  # dispatches events to their target FSMSession.  IO work (LLM calls, tool
-  # calls) must be dispatched via +Runtime.instance.spawn+ or
-  # +BlockingAdapterPool+, then post results back to the loop via
-  # {EventLoop#post}.
-  #
-  # Always active — all Workflow and Agent invocations use the EventLoop.
-  #
-  # == Threading exception (see ADR-010 Rule 2)
-  #
-  # +EventLoop+ is a **deliberate exception** to Phronomy's cooperative-first
-  # concurrency model.  Its dispatch loop is an infinite +while @running+ loop
-  # that must never block the framework's own event processing.
-  # Running it on a shared scheduler task would consume the scheduler, preventing
-  # other tasks from running.  Therefore {#start} creates a dedicated
-  # {Runtime::ThreadScheduler} — this is correct and intentional per ADR-010.
-  # No other framework component should do the same; see the ADR-010 checklist.
-  #
-  # == Handler constraints
-  #
-  # Handlers dispatched by the EventLoop run **on the EventLoop thread**.
-  # They must not:
-  #
-  # * Perform blocking operations directly (database queries, LLM calls, HTTP
-  #   requests).  Schedule blocking work via +Runtime.instance.spawn+ or
-  #   +BlockingAdapterPool+, then post results back with {#post}.
-  # * Call +Workflow#invoke+ (or any synchronous +invoke+) from within a
-  #   handler.  That method would block waiting for the EventLoop to process
-  #   events, causing a deadlock.  Use the async pattern: post a follow-up
-  #   event instead.
-  #
-  # == Fork safety
-  #
-  # +EventLoop.instance+ is lazily initialized. The background thread is not
-  # created until the first call, so Puma worker forking does not duplicate the
-  # thread. No +after_fork+ hook is required.
-  #
-  # == Deadlock warning
-  #
-  # Do NOT call +Workflow#invoke+ (in EventLoop mode) from within a workflow
-  # entry action. The entry action runs on the EventLoop thread; a nested
-  # +invoke+ would block waiting for the same thread to process events →
-  # deadlock. Use the async pattern instead: schedule work via
-  # +Runtime.instance.spawn+ or +BlockingAdapterPool+, then post events back
-  # via +Phronomy::EventLoop.instance.post(...)+.
+  # Runtime-owned FIFO event loop for FSMSession instances.
   class EventLoop
-    # Sentinel target_id for EventLoop management events (:start, :finished, :halted, :error).
-    # Events with this target_id are processed directly by the EventLoop and never
-    # routed to any FSMSession via handle(event).
     SYSTEM_CHANNEL_ID = "__event_loop__"
 
-    # Returns the singleton instance, creating and starting it on first call.
-    def self.instance
-      @instance ||= new.tap(&:start)
-    end
-
-    # Returns true when called from within the EventLoop dispatch task.
-    # Uses a task-local key set by the Runtime-spawned dispatch task so that
-    # the check works correctly for both thread-based and future fiber-based
-    # scheduler backends.
-    # @return [Boolean]
-    # @api private
-    def self.current?
-      Phronomy::Task.current&.name == "event-loop"
-    end
-
-    # Stops and destroys the singleton. Primarily used in tests.
-    # @api private
-    def self.reset!
-      @instance&.stop
-      @instance = nil
-    end
-
-    def initialize
-      @queue = Phronomy::Concurrency::AsyncQueue.new  # global event queue (thread-safe; no Mutex needed)
-      @fsms = {}                  # { id => FSMSession }     — EventLoop thread only
-      @waiting = {}               # { id => completion_queue } — EventLoop thread only
-      # Mutex-backed FSM count for drain-mode shutdown.
-      @fsm_count_mutex = Mutex.new
-      @fsm_count_cond = ConditionVariable.new
-      @fsm_count = 0
-      # Token cancelled when shutdown is requested; new child sessions receive it.
-      @shutdown_token = Phronomy::Concurrency::CancellationToken.new
-      # Fairness metrics (EventLoop thread only, except where noted)
+    QUEUE_BACKLOG_WARNING_THRESHOLD = 1_000
+    QUEUE_BACKLOG_WARNING_INTERVAL_SECONDS = 60.0
+
+    TERMINAL_MANAGEMENT_EVENTS = %i[finished halted error].freeze
+    private_constant :TERMINAL_MANAGEMENT_EVENTS
+
+    STOP = Object.new.freeze
+    private_constant :STOP
+
+    def initialize(runtime:)
+      @runtime = runtime
+      @queue = Phronomy::Concurrency::AsyncQueue.new
+      @queue_metrics_mutex = Mutex.new
+      @queue_depth = 0
+      @max_queue_depth = 0
+      @last_queue_backlog_warning_at = nil
+
+      # @fsms and @waiting are dispatcher-thread-owned.
+      @fsms = {}
+      @waiting = {}
+
+      # Admission is shared by caller threads and the dispatcher. A session ID
+      # enters this set before its :start event is queued and leaves when its
+      # terminal management event is queued.
+      @admitted_session_ids = Set.new
+
+      @lifecycle_mutex = Mutex.new
+      @idle_cond = ConditionVariable.new
+      @shutdown_mutex = Mutex.new
+      @state = :running
+      @outstanding_sessions = 0
+      @cancel_requested = false
+      @shutdown_status = nil
+
       @lag_mutex = Mutex.new
       @last_lag_ns = 0
       @max_lag_ns = 0
       @dispatch_count = 0
       @total_lag_ns = 0
+
+      @task = @runtime.__spawn_event_loop_service { run_loop }
     end
 
-    # Returns the most recently measured event-loop lag in seconds.
-    # Lag is the wall-clock time between {#post} and the moment the event
-    # is dequeued for dispatch.  Thread-safe.
-    # @return [Float]
-    # @api private
     def last_lag_seconds
       @lag_mutex.synchronize { @last_lag_ns } / 1_000_000_000.0
     end
 
-    # Returns the maximum event-loop lag seen since the loop was started.
-    # Thread-safe.
-    # @return [Float]
-    # @api private
     def max_lag_seconds
       @lag_mutex.synchronize { @max_lag_ns } / 1_000_000_000.0
     end
 
-    # Returns the mean event-loop lag across all dispatched events since the
-    # loop was started.  Returns 0.0 when no events have been dispatched.
-    # Thread-safe.
-    # @return [Float]
-    # @api private
     def average_lag_seconds
       @lag_mutex.synchronize do
         return 0.0 if @dispatch_count.zero?
@@ -124,234 +64,407 @@ module Phronomy
       end
     end
 
-    # Registers an FSMSession for execution and returns a completion queue.
-    #
-    # The session and its completion queue are handed off to the EventLoop thread
-    # via the queue payload, so +@fsms+ and +@waiting+ are exclusively written
-    # and read by the EventLoop thread. No Mutex is required.
-    #
-    # The caller blocks on +completion_queue.pop+ to receive the final context
-    # (WorkflowContext) once the workflow finishes or halts. If an error occurred,
-    # the popped value will be an Exception — callers are responsible for re-raising it.
-    #
-    # @param fsm_session [Phronomy::FSMSession]
-    # @return [Phronomy::Concurrency::AsyncQueue] resolves to final/halted context, or an Exception
-    # @api private
+    def queue_depth
+      @queue_metrics_mutex.synchronize { @queue_depth }
+    end
+
+    def max_queue_depth
+      @queue_metrics_mutex.synchronize { @max_queue_depth }
+    end
+
     def register(fsm_session, completion: nil)
-      if Phronomy::EventLoop.current? && !completion.is_a?(Phronomy::Task)
+      if current? && !completion.is_a?(Phronomy::Task)
         raise Phronomy::Error,
-          "Cannot call Workflow#invoke (EventLoop mode) from within an EventLoop " \
-          "entry action. Schedule work via Runtime.instance.spawn or " \
-          "BlockingAdapterPool, then post events back via " \
-          "Phronomy::EventLoop.instance.post(...) instead."
+          "Cannot call a synchronous invocation API from an EventLoop action. " \
+          "Schedule work asynchronously instead."
       end
 
-      completion_queue = completion || Phronomy::Concurrency::AsyncQueue.new
-      # When called from a DeterministicScheduler Fiber (e.g. :fiber backend),
-      # mark the queue so that _pop_cooperative uses track_blocking_await.
-      # This prevents run_until_idle from exiting before the EventLoop thread
-      # (a different OS thread where Scheduler.current is nil) pushes the result.
+      completion_queue =
+        completion || Phronomy::Concurrency::AsyncQueue.new
       scheduler = Phronomy::Runtime::Scheduler.current
-      completion_queue.expect_cross_thread_push(scheduler) if scheduler && completion_queue.respond_to?(:expect_cross_thread_push)
-      # Pass both session and completion_queue in the event payload so that the
-      # EventLoop thread is the sole writer of @fsms and @waiting.
-      # Use SYSTEM_CHANNEL_ID so the management event is never routed to an FSM.
-      @queue.push([Event.new(type: :start, target_id: SYSTEM_CHANNEL_ID,
-        payload: {session: fsm_session, completion: completion_queue}),
-        Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)])
+      if scheduler &&
+          completion_queue.respond_to?(:expect_cross_thread_push)
+        completion_queue.expect_cross_thread_push(scheduler)
+      end
+
+      event = Phronomy::Event.new(
+        type: :start,
+        target_id: SYSTEM_CHANNEL_ID,
+        payload: {
+          session: fsm_session,
+          completion: completion_queue
+        }
+      )
+      queued_depth = nil
+
+      @lifecycle_mutex.synchronize do
+        ensure_accepting_registrations!
+        if @admitted_session_ids.include?(fsm_session.id)
+          raise Phronomy::Error,
+            "FSMSession #{fsm_session.id.inspect} is already registered"
+        end
+
+        @admitted_session_ids.add(fsm_session.id)
+        @outstanding_sessions += 1
+        begin
+          queued_depth = enqueue(
+            [event, monotonic_nanoseconds]
+          )
+        rescue
+          @admitted_session_ids.delete(fsm_session.id)
+          @outstanding_sessions -= 1
+          @idle_cond.broadcast if @outstanding_sessions.zero?
+          raise
+        end
+      end
+
+      check_queue_backlog(queued_depth, event)
       completion_queue
     end
 
-    # Posts an event to the loop. Safe to call from any thread (including IO threads).
-    # The current monotonic clock time is recorded so that the EventLoop can
-    # measure the dispatch lag when it dequeues the event.
-    #
-    # @note **Handler constraint**: do not perform blocking operations or call
-    #   +Workflow#invoke+ directly from within the handler that processes a
-    #   posted event.  Handlers run on the EventLoop thread; blocking there
-    #   stalls all session processing.  For blocking work, post a new event
-    #   after the result is ready.
-    # @param event [Phronomy::Event]
-    # @api private
+    # Internal post operation. Management terminal events close admission for
+    # their session before the terminal event is enqueued.
     def post(event)
-      @queue.push([event, Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)])
+      queued_depth = nil
+      accepted = @lifecycle_mutex.synchronize do
+        next false unless accepting_events?
+
+        terminal_session_id = nil
+        if terminal_management_event?(event)
+          terminal_session_id = event.payload.fetch(:session_id)
+          @admitted_session_ids.delete(terminal_session_id)
+        end
+
+        begin
+          queued_depth = enqueue(
+            [event, monotonic_nanoseconds]
+          )
+        rescue
+          @admitted_session_ids.add(terminal_session_id) if terminal_session_id
+          raise
+        end
+        true
+      end
+      return false unless accepted
+
+      check_queue_backlog(queued_depth, event)
+      true
     end
 
-    # Starts the EventLoop dispatch task under {Runtime} ownership.
+    # Posts an event only when the target session has been admitted and has not
+    # queued a terminal management event. The event remains FIFO with all other
+    # EventLoop work.
     #
-    # The dispatch loop runs as a {Phronomy::Task} so that {Runtime#shutdown}
-    # can drain it together with all other in-flight tasks.  The task is named
-    # +"event-loop"+ so that {.current?} can identify it via
-    # +Task.current&.name+.
-    # @return [self]
-    # @api private
-    def start
-      return self if @task&.alive?
-
-      # Reset shutdown state so the loop can be restarted after a stop.
-      @shutdown_token = Phronomy::Concurrency::CancellationToken.new
-      @fsm_count_mutex.synchronize { @fsm_count = 0 }
-      @running = true
-      # The dispatch loop must always run in a real background thread.
-      # A cooperative scheduler (FakeScheduler/ImmediateBackend) executes tasks
-      # synchronously on the caller's thread, which would block forever inside
-      # the run_loop infinite loop.  Create a dedicated Runtime with
-      # ThreadScheduler to guarantee async execution regardless of the global
-      # runtime_backend setting.
-      thread_runtime = Phronomy::Runtime.new(scheduler: Phronomy::Runtime::ThreadScheduler.new)
-      @task = thread_runtime.spawn(name: "event-loop") do
-        run_loop
+    # A true result reports admission, not transition success.
+    def post_to_session(event)
+      if event.target_id == SYSTEM_CHANNEL_ID
+        raise ArgumentError,
+          "post_to_session cannot target the system channel"
+      end
+
+      queued_depth = nil
+      accepted = @lifecycle_mutex.synchronize do
+        next false unless accepting_events?
+        next false unless @admitted_session_ids.include?(event.target_id)
+
+        queued_depth = enqueue(
+          [event, monotonic_nanoseconds]
+        )
+        true
+      end
+      return false unless accepted
+
+      check_queue_backlog(queued_depth, event)
+      true
+    end
+
+    def admitted_session?(session_id)
+      @lifecycle_mutex.synchronize do
+        @admitted_session_ids.include?(session_id)
+      end
+    end
+
+    def current?
+      Phronomy::Task.current.equal?(@task)
+    end
+
+    def state
+      @lifecycle_mutex.synchronize { @state }
+    end
+
+    def begin_draining
+      @lifecycle_mutex.synchronize do
+        @state = :draining if @state == :running
       end
       self
     end
 
-    # Stops the EventLoop dispatch task.
-    #
-    # Sends a cooperative shutdown sentinel to the event queue so that the
-    # dispatch task can finish any in-flight handler before exiting.  Waits up
-    # to +timeout+ seconds for a clean shutdown; if the task is still alive
-    # afterwards it is cancelled (cooperative cancellation via {Task#cancel!}).
-    #
-    # @param timeout [Numeric] seconds to wait for cooperative shutdown. Defaults
-    #   to +Phronomy.configuration.event_loop_stop_grace_seconds+ (5 s).
-    # @param drain [Boolean] when +true+, wait for all active FSMSessions to
-    #   complete before signalling the loop to stop.  Bounded by +timeout+.
-    #   Defaults to +false+.
-    # @param force_kill [Boolean] deprecated — retained for backward compatibility.
-    #   When +true+, the dispatch task is cancelled via {Task#cancel!} if it does
-    #   not stop within +timeout+.  +Thread#kill+ is no longer used; cooperative
-    #   cancellation (raising {CancellationError}) replaces it.
-    # @return [Symbol] shutdown status:
-    #   - +:clean+ — loop exited cooperatively with no active sessions discarded
-    #   - +:drained_with_discards+ — drain mode requested but sessions remained;
-    #     they were discarded and the loop was stopped
-    #   - +:timeout+ — the task did not stop in time and +force_kill:+ is +false+
-    #   - +:force_killed+ — the task was cancelled because it did not stop in time
-    # @api private
-    def stop(timeout: Phronomy.configuration.event_loop_stop_grace_seconds, drain: false, force_kill: false)
-      @shutdown_token.cancel!
-      status = :clean
-
-      if drain
-        # Wait for active sessions to finish, bounded by timeout.
-        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
-        @fsm_count_mutex.synchronize do
-          while @fsm_count > 0
-            remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
-            break if remaining <= 0
-            @fsm_count_cond.wait(@fsm_count_mutex, remaining)
-          end
-          status = :drained_with_discards if @fsm_count > 0
-        end
+    def idle?
+      @lifecycle_mutex.synchronize do
+        @outstanding_sessions.zero?
       end
+    end
 
-      @running = false
-      @queue.push(:__stop__)   # unblock queue.pop so the task can see @running = false
-      begin
-        @task&.join(timeout)
-      rescue
-        # Task may have terminated with an error (e.g. simulated crash in tests).
-        # Suppress the re-raise so the cleanup below always runs.
-        nil
+    def wait_until_idle(deadline)
+      @lifecycle_mutex.synchronize do
+        until @outstanding_sessions.zero?
+          remaining = deadline - monotonic_now
+          return false if remaining <= 0
+
+          @idle_cond.wait(@lifecycle_mutex, remaining)
+        end
+        true
       end
-      if @task&.alive?
-        if force_kill
-          Phronomy.configuration.logger&.warn(
-            "[Phronomy] EventLoop task did not stop within #{timeout}s; cancelling. " \
-            "This is a last resort — check for blocking operations in event handlers."
-          )
-          @task.cancel!
-          status = :force_killed
-        else
-          Phronomy.configuration.logger&.warn(
-            "[Phronomy] EventLoop task did not stop within #{timeout}s; abandoning " \
-            "(force_kill: false). Check for blocking operations in event handlers."
-          )
-          status = :timeout
+    end
+
+    def shutdown(deadline:, cancel_grace:)
+      @shutdown_mutex.synchronize do
+        return @shutdown_status if @shutdown_status
+
+        if state == :failed
+          join_until(deadline)
+          @shutdown_status = :failed
+          return @shutdown_status
+        end
+
+        begin_draining
+        if wait_until_idle(deadline)
+          begin_stopping_if_idle
+          join_until(deadline)
         end
+
+        @shutdown_status =
+          if task_alive?
+            cancel_and_cleanup(cancel_grace)
+          elsif state == :failed
+            :failed
+          else
+            finalize_terminated(:terminated)
+          end
       end
-      @task = nil
-      status
+    end
+
+    def task_alive?
+      @task&.alive? || false
     end
 
     private
 
     def run_loop
-      while @running
-        item = @queue.pop
-        # :__stop__ is used purely as an unblock signal for @queue.pop; the
-        # actual stop condition is @running == false (set before the push).
-        # Treating it as `next` instead of `break` prevents a stale sentinel
-        # (left by a previous stop call that raced with thread start) from
-        # immediately terminating a freshly restarted EventLoop.
-        next if item == :__stop__
-
-        # item is [event, posted_at_ns] — unwrap and measure lag
+      loop do
+        item = dequeue
+        break if item.equal?(STOP)
+
         event, posted_at_ns = item
-        dequeued_at_ns = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+        dequeued_at_ns = monotonic_nanoseconds
         lag_ns = dequeued_at_ns - posted_at_ns
         update_lag_metrics(lag_ns)
         check_starvation_lag(lag_ns, event)
 
         dispatch_start_ns = dequeued_at_ns
-        if event.target_id == SYSTEM_CHANNEL_ID
-          # Management channel: lifecycle events processed directly by EventLoop.
-          # Never routed to any FSMSession.
-          case event.type
-          when :finished, :halted, :error
-            # session_id is carried in the payload so the FSM can use its own ID
-            # as the FSM-facing target_id for other events.
-            session_id = event.payload[:session_id]
-            @fsms.delete(session_id)
-            cq = @waiting.delete(session_id)
-            complete_waiter(cq, event.payload[:result])
-            # Decrement active FSM count and signal drain waiters.
-            @fsm_count_mutex.synchronize do
-              @fsm_count -= 1
-              @fsm_count_cond.signal if @fsm_count <= 0
-            end
-
-          when :start
-            # session and completion_queue arrive together in the payload so that
-            # this thread is the sole writer of @fsms and @waiting.
-            # completion may be nil for fire-and-forget child sessions (AgentFSM).
-            session = event.payload[:session]
-            cq = event.payload[:completion]
-
-            # When shutdown has been requested, reject new sessions with a
-            # CancellationError rather than starting new LLM calls that would
-            # be interrupted by force-kill.
-            if @shutdown_token.cancelled? && cq
-              complete_waiter(cq, Phronomy::CancellationError.new("EventLoop is shutting down"))
-              next
-            end
-
-            @fsms[session.id] = session
-            @waiting[session.id] = cq if cq
-            @fsm_count_mutex.synchronize { @fsm_count += 1 }
-            session.start
-          end
+        dispatch(event)
+        check_dispatch_time(dispatch_start_ns, event)
+      end
+    rescue Phronomy::CancellationError => error
+      if shutdown_cancel_requested?
+        cleanup_abandoned_work(
+          Phronomy::CancellationError.new(
+            "Runtime shutdown timed out"
+          )
+        )
+      else
+        notify_unexpected_dispatcher_failure(error)
+        raise
+      end
+    rescue => error
+      notify_unexpected_dispatcher_failure(error)
+      raise
+    ensure
+      @lifecycle_mutex.synchronize do
+        @idle_cond.broadcast
+      end
+    end
 
-        else
-          # FSM channel: route to the target FSMSession by target_id.
-          fsm = @fsms[event.target_id]
-          if fsm
-            fsm.handle(event)
-          else
-            # Warn when an event is dropped due to an unknown target_id so that
-            # mis-typed IDs and handler-deregistration races are visible.
-            warn "[Phronomy::EventLoop] Dropped event #{event.type.inspect} — " \
-                 "no handler for target_id #{event.target_id.inspect}"
-          end
+    def dispatch(event)
+      if event.target_id == SYSTEM_CHANNEL_ID
+        dispatch_management(event)
+        return
+      end
+
+      fsm = @fsms[event.target_id]
+      if fsm
+        fsm.handle(event)
+      else
+        warn(
+          "[Phronomy::EventLoop] Dropped event #{event.type.inspect} — " \
+          "no handler for target_id #{event.target_id.inspect}"
+        )
+      end
+    end
+
+    def dispatch_management(event)
+      case event.type
+      when :finished, :halted, :error
+        session_id = event.payload.fetch(:session_id)
+        session = @fsms.delete(session_id)
+        waiter = @waiting.delete(session_id)
+        complete_waiter(
+          waiter,
+          event.payload.fetch(:result)
+        )
+        decrement_outstanding if session
+      when :start
+        session = event.payload.fetch(:session)
+        waiter = event.payload[:completion]
+        @fsms[session.id] = session
+        @waiting[session.id] = waiter if waiter
+        session.start
+      end
+    end
+
+    def terminal_management_event?(event)
+      event.target_id == SYSTEM_CHANNEL_ID &&
+        TERMINAL_MANAGEMENT_EVENTS.include?(event.type) &&
+        event.payload.is_a?(Hash) &&
+        event.payload.key?(:session_id)
+    end
+
+    def begin_stopping_if_idle
+      @lifecycle_mutex.synchronize do
+        return false unless @state == :draining
+        return false unless @outstanding_sessions.zero?
+
+        @state = :stopping
+        enqueue(STOP)
+        true
+      end
+    end
+
+    def cancel_and_cleanup(cancel_grace)
+      task = @task
+      @lifecycle_mutex.synchronize do
+        @state = :stopping unless @state == :failed
+        @cancel_requested = true
+      end
+
+      task&.cancel!
+      begin
+        task&.join(cancel_grace)
+      rescue
+        nil
+      end
+
+      if task&.alive?
+        @lifecycle_mutex.synchronize do
+          @state = :failed
         end
+        return :cancel_timeout
+      end
 
-        # Check how long this dispatch took; warn if it exceeds the threshold.
-        check_dispatch_time(dispatch_start_ns, event)
+      return :failed if state == :failed
+
+      cleanup_abandoned_work(
+        Phronomy::CancellationError.new(
+          "Runtime shutdown timed out"
+        )
+      )
+      finalize_terminated(:cancelled)
+    end
+
+    def cleanup_abandoned_work(error)
+      drain_queued_items.each do |item|
+        next if item.equal?(STOP)
+
+        event, = item
+        next unless event.target_id == SYSTEM_CHANNEL_ID
+        next unless event.type == :start
+
+        complete_waiter(
+          event.payload[:completion],
+          error
+        )
       end
-    rescue => e
-      # Unblock all waiting callers if the loop dies unexpectedly.
-      @waiting.values.each { |cq| complete_waiter(cq, e) }
-      raise
+
+      @waiting.values.each do |waiter|
+        complete_waiter(waiter, error)
+      end
+      @waiting.clear
+      @fsms.clear
+      @lifecycle_mutex.synchronize do
+        @admitted_session_ids.clear
+        @outstanding_sessions = 0
+        @idle_cond.broadcast
+      end
+    end
+
+    def drain_queued_items
+      items = []
+      loop do
+        item = dequeue(timeout: 0)
+        break unless item
+
+        items << item
+      end
+      items
+    end
+
+    def notify_unexpected_dispatcher_failure(error)
+      @lifecycle_mutex.synchronize do
+        @state = :failed
+        @admitted_session_ids.clear
+        @idle_cond.broadcast
+      end
+      @waiting.values.each do |waiter|
+        complete_waiter(waiter, error)
+      end
+      @runtime.__event_loop_failed(error)
+    end
+
+    def shutdown_cancel_requested?
+      @lifecycle_mutex.synchronize do
+        @cancel_requested && @state == :stopping
+      end
+    end
+
+    def accepting_events?
+      %i[running draining].include?(@state)
+    end
+
+    def ensure_accepting_registrations!
+      return if accepting_events?
+
+      raise Phronomy::RuntimeShutdownError,
+        "EventLoop is #{@state}; new sessions are not accepted"
+    end
+
+    def decrement_outstanding
+      @lifecycle_mutex.synchronize do
+        if @outstanding_sessions.positive?
+          @outstanding_sessions -= 1
+        end
+        @idle_cond.broadcast if @outstanding_sessions.zero?
+      end
+    end
+
+    def join_until(deadline)
+      remaining = deadline - monotonic_now
+      return if remaining <= 0
+
+      begin
+        @task&.join(remaining)
+      rescue
+        nil
+      end
+    end
+
+    def finalize_terminated(status)
+      @lifecycle_mutex.synchronize do
+        @state = :terminated
+        @admitted_session_ids.clear
+        @task = nil unless @task&.alive?
+        @idle_cond.broadcast
+      end
+      status
     end
 
     def complete_waiter(waiter, payload)
@@ -363,13 +476,84 @@ module Phronomy
           waiter.transition!(:failed, error: payload)
         else
           waiter.backend.unblock(payload, nil)
-          waiter.transition!(:completed, value: payload)
+          waiter.transition!(
+            :completed,
+            value: payload
+          )
         end
       else
         waiter.push(payload)
       end
     end
 
+    def enqueue(item)
+      depth = @queue_metrics_mutex.synchronize do
+        @queue_depth += 1
+        if @queue_depth > @max_queue_depth
+          @max_queue_depth = @queue_depth
+        end
+        @queue_depth
+      end
+      @queue.push(item)
+      depth
+    rescue
+      @queue_metrics_mutex.synchronize do
+        @queue_depth -= 1 if @queue_depth.positive?
+      end
+      raise
+    end
+
+    def dequeue(timeout: nil)
+      item = nil
+      begin
+        item = @queue.pop(timeout: timeout)
+      ensure
+        if item
+          @queue_metrics_mutex.synchronize do
+            @queue_depth -= 1 if @queue_depth.positive?
+          end
+        end
+      end
+      item
+    end
+
+    def check_queue_backlog(depth, event)
+      return unless depth >= QUEUE_BACKLOG_WARNING_THRESHOLD
+
+      now = monotonic_now
+      max_depth = nil
+      should_warn = @queue_metrics_mutex.synchronize do
+        last = @last_queue_backlog_warning_at
+        if last &&
+            (now - last) <
+                QUEUE_BACKLOG_WARNING_INTERVAL_SECONDS
+          next false
+        end
+
+        @last_queue_backlog_warning_at = now
+        max_depth = @max_queue_depth
+        true
+      end
+      return unless should_warn
+
+      warn_queue_backlog(
+        "[Phronomy::EventLoop] Queue backlog is high: " \
+        "depth=#{depth} max_depth=#{max_depth} " \
+        "threshold=#{QUEUE_BACKLOG_WARNING_THRESHOLD} " \
+        "event=#{event.type.inspect} " \
+        "target_id=#{event.target_id.inspect}. " \
+        "Events are not dropped; inspect slow callbacks or " \
+        "high streaming concurrency."
+      )
+    end
+
+    def warn_queue_backlog(message)
+      logger = Phronomy.configuration.logger
+      logger ? logger.warn(message) : Kernel.warn(message)
+    rescue
+      nil
+    end
+
     def update_lag_metrics(lag_ns)
       @lag_mutex.synchronize do
         @last_lag_ns = lag_ns
@@ -380,30 +564,50 @@ module Phronomy
     end
 
     def check_starvation_lag(lag_ns, event)
-      threshold = Phronomy.configuration.event_loop_starvation_threshold_seconds
-      return unless threshold && lag_ns > (threshold * 1_000_000_000)
+      threshold =
+        Phronomy.configuration
+          .event_loop_starvation_threshold_seconds
+      return unless threshold
+      return unless lag_ns > (threshold * 1_000_000_000)
 
       Phronomy.configuration.logger&.warn do
-        "[Phronomy::EventLoop] Starvation detected: event #{event.type.inspect} " \
-        "for target #{event.target_id.inspect} waited " \
-        "#{format("%.3f", lag_ns / 1_000_000_000.0)}s in queue " \
-        "(threshold: #{threshold}s)"
+        "[Phronomy::EventLoop] Starvation detected: " \
+          "event #{event.type.inspect} " \
+          "for target #{event.target_id.inspect} waited " \
+          "#{format("%.3f", lag_ns / 1_000_000_000.0)}s " \
+          "in queue (threshold: #{threshold}s)"
       end
     end
 
     def check_dispatch_time(dispatch_start_ns, event)
-      threshold = Phronomy.configuration.event_loop_dispatch_threshold_seconds
+      threshold =
+        Phronomy.configuration
+          .event_loop_dispatch_threshold_seconds
       return unless threshold
 
-      elapsed_ns = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond) - dispatch_start_ns
-      return unless elapsed_ns > (threshold * 1_000_000_000)
+      elapsed_ns = monotonic_nanoseconds - dispatch_start_ns
+      return unless elapsed_ns >
+        (threshold * 1_000_000_000)
 
       Phronomy.configuration.logger&.warn do
-        "[Phronomy::EventLoop] Long dispatch: event #{event.type.inspect} " \
-        "for target #{event.target_id.inspect} took " \
-        "#{format("%.3f", elapsed_ns / 1_000_000_000.0)}s on the EventLoop thread " \
-        "(threshold: #{threshold}s). Consider moving blocking work to BlockingAdapterPool."
+        "[Phronomy::EventLoop] Long dispatch: " \
+          "event #{event.type.inspect} " \
+          "for target #{event.target_id.inspect} took " \
+          "#{format("%.3f", elapsed_ns / 1_000_000_000.0)}s " \
+          "on the EventLoop thread (threshold: #{threshold}s). " \
+          "Consider moving blocking work to BlockingAdapterPool."
       end
     end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def monotonic_nanoseconds
+      Process.clock_gettime(
+        Process::CLOCK_MONOTONIC,
+        :nanosecond
+      )
+    end
   end
 end