julewire-ractor 1.0.0 → 1.1.0

data/lib/julewire/ractor/destination.rb

@@ -6,19 +6,51 @@ require "concurrent/atomic/atomic_reference"
 module Julewire
   module Ractor
     class Destination # rubocop:disable Metrics/ClassLength -- Owns parent queue, worker lifecycle, and health.
+      class QueueSlots
+        def initialize(max_queue:)
+          @max_queue = max_queue
+          @in_flight = Concurrent::AtomicFixnum.new
+        end
+
+        def value = @in_flight.value
+
+        def reserve
+          return true unless @max_queue.positive?
+
+          result = {}
+          @in_flight.update do |current|
+            result[:reserved] = current < @max_queue
+            result.fetch(:reserved) ? current + 1 : current
+          end
+          result.fetch(:reserved)
+        end
+
+        def release
+          return false unless @max_queue.positive?
+
+          result = {}
+          @in_flight.update do |current|
+            result[:underflow] = current.zero?
+            current.positive? ? current - 1 : current
+          end
+          result.fetch(:underflow)
+        end
+      end
+
       COUNTER_KEYS = %i[
         closed_dropped
         queue_full_dropped
         queued
         received
         send_error
+        slot_underflow_ignored
         worker_accepted
         worker_dropped
       ].freeze
       DEFAULT_MAX_QUEUE = 1024
       DEFAULT_REQUEST_TIMEOUT = 1
-      TIMEOUT_THREAD_NAME = "julewire-ractor-destination-timeout"
-      private_constant :COUNTER_KEYS
+      WORKER_STOP_MESSAGE = { command: :close_worker }.freeze
+      private_constant :COUNTER_KEYS, :QueueSlots, :WORKER_STOP_MESSAGE
 
       attr_reader :name
 
@@ -41,6 +73,8 @@ module Julewire
         Core::Validation.validate_byte_limit!(max_record_bytes, name: :max_record_bytes)
         Core::Validation.validate_non_negative_integer!(max_queue, name: :max_queue)
         Core::Validation.validate_timeout!(request_timeout, name: :request_timeout)
+        raise ArgumentError, "request_timeout must be a non-negative finite Numeric" if request_timeout.nil?
+
         Core::Validation.validate_callable!(on_drop, name: :on_drop, allow_nil: true)
         Core::Validation.validate_callable!(on_failure, name: :on_failure, allow_nil: true)
         @output = output
@@ -50,7 +84,6 @@ module Julewire
         @request_timeout = request_timeout
         @on_drop = on_drop
         @on_failure = on_failure
-        @scheduler = Core::Scheduling::DeadlineScheduler.new(thread_name: TIMEOUT_THREAD_NAME, idle: :exit)
         initialize_tracking
         start_worker
       end
@@ -58,33 +91,37 @@ module Julewire
       def emit(record)
         increment(:received)
         return drop(:closed_dropped, record) if closed?
-        return drop(:queue_full_dropped, record) if queue_full?
+        return drop(:queue_full_dropped, record) unless @queue_slots.reserve
 
-        @port.send({ command: :emit, record: record })
-        @in_flight.increment
-        increment(:queued)
+        begin
+          degradation_marker = @health.degradation_marker
+          @port.send({ command: :emit, degradation_marker: degradation_marker, record: record })
+          increment(:queued)
+        rescue StandardError => e
+          release_slot
+          record_failure(e, phase: :ractor_send)
+          drop(:send_error, record)
+        end
         nil
-      rescue StandardError => e
-        record_failure(e, phase: :ractor_send)
-        drop(:send_error, record)
       end
 
       def flush(timeout: nil)
-        timeout = @request_timeout if timeout.nil?
-        request(:flush, timeout: timeout)
+        @health.recover_if_successful { request(:flush, timeout: lifecycle_timeout(timeout)) }
       end
 
       def close(timeout: nil)
-        timeout = @request_timeout if timeout.nil?
+        timeout = lifecycle_timeout(timeout)
         @closed.set(true)
-        result = request(:close, timeout: timeout)
-        close_ports
+        result = request(:close, timeout: timeout, allow_closed: true)
+        close_ports(timeout: timeout)
         result
       end
 
       def after_fork!
-        close_ports
-        @scheduler.after_fork!
+        if (@process_id - Process.pid).zero? && !close_ports(timeout: @request_timeout)
+          raise Core::Error, "ractor destination worker did not stop within #{@request_timeout} seconds"
+        end
+
         initialize_tracking
         start_worker
         self
@@ -97,11 +134,17 @@ module Julewire
 
       def health
         worker = request(:health, timeout: @request_timeout)
-        worker = @worker_health.get unless worker.is_a?(Hash)
-        @worker_health.set(worker) if worker.is_a?(Hash)
+        if worker.equal?(false)
+          worker = @worker_health.get
+        else
+          raise TypeError, "ractor destination health must be a Hash" unless worker.instance_of?(Hash)
+
+          Core::Integration::Protocol.validate_symbol_keys(worker)
+          @worker_health.set(worker)
+        end
 
         @health.snapshot(
-          in_flight: @in_flight.value,
+          in_flight: @queue_slots.value,
           max_queue: @max_queue,
           status: status_for(worker),
           worker: worker
@@ -116,22 +159,28 @@ module Julewire
       end
 
       def initialize_tracking
-        @closed = Concurrent::AtomicReference.new(false)
+        @process_id = Process.pid
+        @scheduler = ReplyTimeoutScheduler.new(timeout_value: false)
+        @closed = Concurrent::AtomicReference.new
         @health = Core::Integration::DestinationHealth.new(counter_keys: COUNTER_KEYS, failure_counter: nil)
-        @in_flight = Concurrent::AtomicFixnum.new(0)
+        @queue_slots = QueueSlots.new(max_queue: @max_queue)
         @worker_health = Concurrent::AtomicReference.new
       end
 
       def start_worker
         @ack_port = ::Ractor::Port.new
-        setup_port = ::Ractor::Port.new
-        @worker = spawn_worker(setup_port)
-        @port = receive_worker_port(setup_port)
+        PortLifecycle.with_port do |setup_port|
+          @worker = spawn_worker(setup_port)
+          @port = WorkerHandshake.receive(
+            setup_port: setup_port,
+            worker: @worker,
+            scheduler: @scheduler
+          )
+        end
         @ack_thread = start_ack_thread
-      rescue StandardError => e
-        raise ArgumentError, "ractor destination collaborators must be ractor-copyable or shareable: #{e.message}"
-      ensure
-        PortLifecycle.close(setup_port) if defined?(setup_port) && setup_port
+      rescue TypeError, ::Ractor::Error => e
+        @port = @worker = @ack_port = @ack_thread = nil
+        raise ArgumentError, "ractor destination collaborators must be ractor-copyable or shareable: #{e}"
       end
 
       def spawn_worker(setup_port)
@@ -142,7 +191,7 @@ module Julewire
         max_record_bytes = @max_record_bytes
         close_output = @close_output
 
-        # :nocov:
+        # simplecov:disable
         ::Ractor.new(setup_port, ack_port, formatter, encoder, output, max_record_bytes, close_output,
                      name: "julewire-ractor-destination") do |worker_port, worker_ack_port, worker_formatter,
                                                              worker_encoder, worker_output, worker_max_record_bytes,
@@ -159,21 +208,19 @@ module Julewire
             close_output: worker_close_output
           )
         end
-        # :nocov:
-      end
-
-      def receive_worker_port(setup_port)
-        selected, value = ::Ractor.select(setup_port, @worker)
-        return value if selected.equal?(setup_port) && value.is_a?(::Ractor::Port)
-
-        raise ArgumentError, "ractor destination worker did not start"
+        # simplecov:enable
       end
 
       def start_ack_thread
+        ack_port = @ack_port
         thread = Thread.new do
           loop do
-            message = @ack_port.receive
-            break if message.is_a?(Hash) && message[:event] == :closed
+            message = ack_port.receive
+            Core::Integration::Protocol.validate_symbol_hash(message)
+            event = message.fetch(:event)
+            unless event == :ack
+              raise ArgumentError, "unknown ractor destination acknowledgement event: #{event.inspect}"
+            end
 
             handle_ack(message)
           end
@@ -181,65 +228,86 @@ module Julewire
           record_failure(e, phase: :ack)
         end
         thread.name = "julewire-ractor-destination-ack"
-        thread.report_on_exception = true
         thread
       end
 
       def handle_ack(message)
-        return unless message.is_a?(Hash) && message[:event] == :ack
-
-        decrement_in_flight
-        case message[:status]
+        release_slot
+        case message.fetch(:status)
         when :accepted
           increment(:worker_accepted)
+          @health.clear_degradation_if_unchanged(message.fetch(:degradation_marker))
         when :dropped
           increment(:worker_dropped)
+        else
+          raise ArgumentError, "unknown ractor destination acknowledgement status: #{message.fetch(:status).inspect}"
         end
       end
 
-      def request(command, timeout:)
-        return false if closed? && command != :health && command != :close
+      def request(command, timeout:, allow_closed: false)
+        return false if closed? && !allow_closed
 
-        reply = ::Ractor::Port.new
-        @port.send({ command: command, reply: reply })
-        wait_for_reply(reply, timeout)
+        PortLifecycle.with_port do |reply|
+          @port.send({ command: command, reply: reply })
+          wait_for_reply(reply, timeout, command)
+        end
       rescue StandardError => e
         record_failure(e, phase: :request, command: command)
         false
-      ensure
-        PortLifecycle.close(reply) if reply
       end
 
-      def wait_for_reply(reply, timeout)
-        return reply.receive unless timeout
+      def wait_for_reply(reply, timeout, command)
+        @scheduler.with_timeout(reply, timeout: timeout) do
+          selected, value = ::Ractor.select(reply, @worker)
+          return value if selected.equal?(reply)
 
-        timeout_port = ::Ractor::Port.new
-        token = @scheduler.schedule(timeout) do
-          timeout_port.send(:timeout)
-        rescue StandardError
-          nil
+          raise Core::Error, "ractor destination worker stopped before #{command} replied"
         end
-        selected, response = ::Ractor.select(reply, timeout_port)
-
-        selected.equal?(timeout_port) ? false : response
-      ensure
-        @scheduler.cancel(token) if defined?(token)
-        PortLifecycle.close(timeout_port) if defined?(timeout_port) && timeout_port
+      rescue ::Ractor::RemoteError
+        raise Core::Error, "ractor destination worker stopped before #{command} replied"
       end
 
-      def queue_full?
-        @max_queue.positive? && @in_flight.value >= @max_queue
+      def closed? = @closed.get
+
+      def close_ports(timeout:)
+        worker_stopped = true
+        begin
+          if @worker
+            begin
+              @port.send(WORKER_STOP_MESSAGE)
+            rescue ::Ractor::ClosedError
+              # A stopped worker no longer accepts commands. Still collect it
+              # below so an abnormal exit remains observable.
+            end
+            worker_stopped = wait_for_worker(timeout)
+          end
+        rescue ::Ractor::RemoteError => e
+          record_failure(e, phase: :worker_stop)
+          worker_stopped = true
+        ensure
+          @ack_thread&.kill
+          @ack_thread&.join
+          PortLifecycle.close(@ack_port)
+          @port = @worker = nil if worker_stopped
+          @ack_port = nil
+          @ack_thread = nil
+        end
+
+        worker_stopped
       end
 
-      def closed? = @closed.get
+      def wait_for_worker(timeout)
+        PortLifecycle.with_port do |timeout_port|
+          @scheduler.with_timeout(timeout_port, timeout: timeout) do
+            selected, = ::Ractor.select(@worker, timeout_port)
+            selected.equal?(@worker)
+          end
+        end
+      end
 
-      def close_ports
-        @port&.send({ command: :close_worker }) unless @port&.closed?
-      rescue StandardError
-        nil
-      ensure
-        PortLifecycle.close(@port) if @port
-        PortLifecycle.close(@ack_port) if @ack_port
+      def lifecycle_timeout(timeout)
+        Core::Validation.validate_timeout!(timeout, name: :timeout)
+        timeout || @request_timeout
       end
 
       def drop(reason, record)
@@ -253,24 +321,23 @@ module Julewire
       end
 
       def record_loss(reason, record)
-        metadata = Core::Records::Metadata.call(record)
         @health.record_loss(
           reason: reason,
-          event: metadata[:event],
-          severity: metadata[:severity],
-          source: metadata[:source]
+          event: record.event,
+          severity: record.severity,
+          source: record.source
         )
       end
 
       def record_failure(error, **metadata)
-        @health.record_failure(error, counter: nil, **metadata)
+        @health.record_failure(error, **metadata)
         Core::Diagnostics::CallbackNotifier.call(@on_failure, error, { destination: name }.merge(metadata))
-      rescue StandardError
-        nil
       end
 
-      def decrement_in_flight
-        @in_flight.decrement if @in_flight.value.positive?
+      def release_slot
+        # Late or duplicate ACKs can arrive after teardown/reset; keep them visible
+        # without treating the ignored underflow as an operator-facing defect.
+        increment(:slot_underflow_ignored) if @queue_slots.release
       end
 
       def increment(key)
@@ -279,10 +346,8 @@ module Julewire
 
       def status_for(worker)
         return :closed if closed?
-        return :degraded if @health.degraded?
-        return :degraded if worker.is_a?(Hash) && worker[:status] == :degraded
 
-        :ok
+        :degraded if worker&.fetch(:status) == :degraded
       end
     end
   end

data/lib/julewire/ractor/destination_worker.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# :nocov:
 module Julewire
   module Ractor
     class DestinationWorker
@@ -47,27 +46,24 @@ module Julewire
         @ack_port = ack_port
         loop do
           message = command_port.receive
-          break if close_message?(message)
+          current_command = command(message)
+          break if current_command == :close_worker
 
-          break if dispatch(message) == :close
+          break if dispatch(message, current_command) == :close
         end
       ensure
         close_output
-        ack(:closed)
       end
 
       private
 
-      def close_message?(message)
-        message.is_a?(Hash) && message[:command] == :close_worker
-      end
-
-      def dispatch(message)
-        return unless message.is_a?(Hash)
-
-        case message[:command]
+      def dispatch(message, current_command)
+        case current_command
         when :emit
-          emit(message[:record])
+          emit(
+            message.fetch(:record),
+            parent_degradation_marker: message.fetch(:degradation_marker)
+          )
         when :flush
           reply_to(message, call_output_lifecycle(:flush))
         when :close
@@ -75,51 +71,47 @@ module Julewire
           :close
         when :health
           reply_to(message, health)
+        else
+          raise ArgumentError, "unknown ractor destination command: #{current_command.inspect}"
         end
-      rescue StandardError => e
-        record_failure(e, phase: :dispatch)
-        reply_to(message, false)
       end
 
-      def emit(record)
-        ack(@write_step.call(record) == :accepted ? :accepted : :dropped)
-      rescue StandardError => e
-        record_failure(e, phase: :emit)
-        ack(:dropped)
+      def command(message)
+        Core::Integration::Protocol.validate_symbol_hash(message)
+        message.fetch(:command)
+      end
+
+      def emit(record, parent_degradation_marker:)
+        accepted = @health.recover_if_successful { @write_step.call(record) }
+        ack(accepted ? :accepted : :dropped, degradation_marker: parent_degradation_marker)
       end
 
       def call_output_lifecycle(method_name)
-        return close_lifecycle if method_name == :close
-        return true unless @output.respond_to?(method_name)
+        return close_lifecycle_succeeds? if method_name == :close
 
-        @output.public_send(method_name) != false
+        @health.recover_if_successful do
+          !@output.respond_to?(method_name) || @output.public_send(method_name) != false
+        end
       rescue StandardError => e
         record_failure(e, phase: :output_lifecycle, action: method_name)
         false
       end
 
-      def close_lifecycle
-        return true if output_closed?
+      def close_lifecycle_succeeds?
+        return true if @output.respond_to?(:closed?) && @output.closed?
         return @output.close != false if @close_output && @output.respond_to?(:close)
         return @output.flush != false if @output.respond_to?(:flush)
 
         true
-      rescue StandardError => e
-        record_failure(e, phase: :output_lifecycle, action: :close)
-        false
       end
 
       def close_output
-        return if output_closed?
+        return if @output.respond_to?(:closed?) && @output.closed?
         return unless @close_output && @output.respond_to?(:close)
 
         @output.close
-      rescue StandardError => e
-        record_failure(e, phase: :output_lifecycle, action: :close)
-      end
-
-      def output_closed?
-        @output.respond_to?(:closed?) ? @output.closed? : false
+      rescue StandardError
+        nil
       end
 
       def health
@@ -131,7 +123,7 @@ module Julewire
       end
 
       def record_failure(error, **metadata)
-        @health.record_failure(error, counter: nil, **metadata)
+        @health.record_failure(error, **metadata)
       end
 
       def record_loss(reason, **metadata)
@@ -139,7 +131,7 @@ module Julewire
       end
 
       def record_write_step_failure(error, metadata)
-        record_failure(error, **recordless_metadata(metadata))
+        record_failure(error, **metadata)
       end
 
       def record_write_step_loss(reason, metadata)
@@ -156,15 +148,19 @@ module Julewire
         metadata.except(:record)
       end
 
-      def ack(status)
-        @ack_port.send({ event: :ack, status: status })
-      rescue StandardError
-        nil
+      def ack(status, degradation_marker:)
+        @ack_port.send({ degradation_marker: degradation_marker, event: :ack, status: status })
       end
 
       def reply_to(message, response)
-        reply = message[:reply]
-        reply.send(response) if reply.is_a?(::Ractor::Port)
+        reply = message.fetch(:reply)
+        raise TypeError, "ractor destination reply must be a Ractor::Port" unless reply.is_a?(::Ractor::Port)
+
+        send_reply(reply, response)
+      end
+
+      def send_reply(reply, response)
+        reply.send(response)
       rescue StandardError => e
         record_failure(e, phase: :reply)
       end
@@ -173,4 +169,3 @@ module Julewire
     private_constant :DestinationWorker
   end
 end
-# :nocov:

data/lib/julewire/ractor/fanout.rb

@@ -7,7 +7,9 @@ module Julewire
 
       def initialize(destinations:, name: :ractor_fanout, on_failure: nil)
         @name = Core::Destinations.normalize_name(name)
-        @destinations = Array(destinations).map { normalize_destination(it) }.freeze
+        raise ArgumentError, "destinations must be an Array" unless destinations.instance_of?(Array)
+
+        @destinations = destinations.map { normalize_destination(it) }
         raise ArgumentError, "destinations must not be empty" if @destinations.empty?
 
         Core::Validation.validate_callable!(on_failure, name: :on_failure, allow_nil: true)
@@ -16,7 +18,9 @@ module Julewire
       end
 
       def emit(record)
-        @destinations.each { emit_to_destination(it, record) }
+        @health.recover_if_successful do
+          @destinations.each { emit_to_destination(it, record) }
+        end
         nil
       end
 
@@ -29,10 +33,12 @@ module Julewire
       end
 
       def after_fork!
-        @destinations.each do |destination|
-          destination.after_fork! if destination.respond_to?(:after_fork!)
-        rescue StandardError => e
-          record_failure(e, action: :after_fork, destination: destination.name)
+        @health.recover_if_successful do
+          @destinations.each do |destination|
+            destination.after_fork! if destination.respond_to?(:after_fork!)
+          rescue StandardError => e
+            record_failure(e, action: :after_fork, destination: destination.name)
+          end
         end
         self
       end
@@ -61,14 +67,16 @@ module Julewire
       end
 
       def call_lifecycle(method_name, timeout:)
-        ok = true
-        @destinations.each do |destination|
-          ok = false if destination.public_send(method_name, timeout: timeout) == false
-        rescue StandardError => e
-          record_failure(e, action: method_name, destination: destination.name)
-          ok = false
+        @health.recover_if_successful do
+          ok = true
+          @destinations.each do |destination|
+            ok = false if destination.public_send(method_name, timeout: timeout) == false
+          rescue StandardError => e
+            record_failure(e, action: method_name, destination: destination.name)
+            ok = false
+          end
+          ok
         end
-        ok
       end
 
       def destination_health(destination)
@@ -78,17 +86,16 @@ module Julewire
       end
 
       def health_status(destinations)
-        return :degraded if @health.last_failure
-        return :degraded if destinations.any? { |_name, health| health[:status] == :degraded || health[:phase] }
-
-        :ok
+        :degraded if destinations.any? { |_name, health| health[:status] == :degraded || health[:phase] }
       end
 
       def record_failure(error, **metadata)
-        @health.record_failure(error, counter: nil, phase: :ractor_fanout, **metadata)
-        @on_failure&.call(error, **metadata, phase: :ractor_fanout)
-      rescue StandardError
-        nil
+        @health.record_failure(error, phase: :ractor_fanout, **metadata)
+        callback_result = Core::Diagnostics::CallbackNotifier.call(@on_failure, error,
+                                                                   metadata.merge(phase: :ractor_fanout))
+        return unless Core::Diagnostics::CallbackNotifier.failure?(callback_result)
+
+        @health.record_callback_failure(callback_result)
       end
     end
   end

data/lib/julewire/ractor/port_lifecycle.rb

@@ -4,11 +4,17 @@ module Julewire
   module Ractor
     module PortLifecycle
       class << self
+        def with_port(port = ::Ractor::Port.new)
+          yield port
+        ensure
+          close(port)
+        end
+
         def close(port)
           return unless port.respond_to?(:close)
-          return if port.respond_to?(:closed?) && port.closed?
 
           port.close
+          nil
         rescue StandardError
           nil
         end