julewire-ractor 1.0.0

data/lib/julewire/ractor/child_stats.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Ractor
+    class ChildStats
+      COUNTER_KEYS = %i[
+        messages_dropped
+        messages_sent
+        requests_failed
+        requests_sent
+        requests_timed_out
+      ].freeze
+      private_constant :COUNTER_KEYS
+
+      def initialize
+        @mutex = Mutex.new
+        @counters = COUNTER_KEYS.to_h { [it, 0] }
+      end
+
+      def message_sent = increment(:messages_sent)
+
+      def message_dropped(error)
+        record_error(:messages_dropped, error)
+      end
+
+      def request_sent = increment(:requests_sent)
+
+      def request_failed(error)
+        record_error(:requests_failed, error)
+      end
+
+      def request_timed_out = increment(:requests_timed_out)
+
+      def reset!
+        @mutex.synchronize do
+          @counters.each_key { @counters[it] = 0 }
+          @last_error_class = nil
+        end
+        nil
+      end
+
+      def to_h
+        @mutex.synchronize do
+          {
+            counts: @counters.dup.freeze,
+            last_error_class: @last_error_class
+          }.compact.freeze
+        end
+      end
+
+      private
+
+      def increment(key)
+        @mutex.synchronize { @counters[key] += 1 }
+        nil
+      end
+
+      def record_error(key, error)
+        @mutex.synchronize do
+          @counters[key] += 1
+          @last_error_class = error.class.name
+        end
+        nil
+      end
+    end
+
+    private_constant :ChildStats
+  end
+end

data/lib/julewire/ractor/destination.rb

@@ -0,0 +1,289 @@
+# frozen_string_literal: true
+
+require "concurrent/atomic/atomic_fixnum"
+require "concurrent/atomic/atomic_reference"
+
+module Julewire
+  module Ractor
+    class Destination # rubocop:disable Metrics/ClassLength -- Owns parent queue, worker lifecycle, and health.
+      COUNTER_KEYS = %i[
+        closed_dropped
+        queue_full_dropped
+        queued
+        received
+        send_error
+        worker_accepted
+        worker_dropped
+      ].freeze
+      DEFAULT_MAX_QUEUE = 1024
+      DEFAULT_REQUEST_TIMEOUT = 1
+      TIMEOUT_THREAD_NAME = "julewire-ractor-destination-timeout"
+      private_constant :COUNTER_KEYS
+
+      attr_reader :name
+
+      def initialize( # rubocop:disable Metrics/ParameterLists -- Destination setup mirrors core destination knobs.
+        output:,
+        name: :ractor,
+        formatter: Julewire::RecordFormatter.new,
+        encoder: Julewire::JsonEncoder.new,
+        max_record_bytes: Core::DEFAULT_MAX_RECORD_BYTES,
+        max_queue: DEFAULT_MAX_QUEUE,
+        close_output: false,
+        request_timeout: DEFAULT_REQUEST_TIMEOUT,
+        on_drop: nil,
+        on_failure: nil
+      )
+        @name = Core::Destinations.normalize_name(name)
+        @formatter = validate_callable(formatter, name: :formatter)
+        @encoder = validate_callable(encoder, name: :encoder)
+        Core::Destinations::Sink.validate_writeable!(output)
+        Core::Validation.validate_byte_limit!(max_record_bytes, name: :max_record_bytes)
+        Core::Validation.validate_non_negative_integer!(max_queue, name: :max_queue)
+        Core::Validation.validate_timeout!(request_timeout, name: :request_timeout)
+        Core::Validation.validate_callable!(on_drop, name: :on_drop, allow_nil: true)
+        Core::Validation.validate_callable!(on_failure, name: :on_failure, allow_nil: true)
+        @output = output
+        @max_record_bytes = max_record_bytes
+        @max_queue = max_queue
+        @close_output = close_output
+        @request_timeout = request_timeout
+        @on_drop = on_drop
+        @on_failure = on_failure
+        @scheduler = Core::Scheduling::DeadlineScheduler.new(thread_name: TIMEOUT_THREAD_NAME, idle: :exit)
+        initialize_tracking
+        start_worker
+      end
+
+      def emit(record)
+        increment(:received)
+        return drop(:closed_dropped, record) if closed?
+        return drop(:queue_full_dropped, record) if queue_full?
+
+        @port.send({ command: :emit, record: record })
+        @in_flight.increment
+        increment(:queued)
+        nil
+      rescue StandardError => e
+        record_failure(e, phase: :ractor_send)
+        drop(:send_error, record)
+      end
+
+      def flush(timeout: nil)
+        timeout = @request_timeout if timeout.nil?
+        request(:flush, timeout: timeout)
+      end
+
+      def close(timeout: nil)
+        timeout = @request_timeout if timeout.nil?
+        @closed.set(true)
+        result = request(:close, timeout: timeout)
+        close_ports
+        result
+      end
+
+      def after_fork!
+        close_ports
+        @scheduler.after_fork!
+        initialize_tracking
+        start_worker
+        self
+      rescue StandardError => e
+        record_failure(e, phase: :after_fork)
+        self
+      end
+
+      def resource_identity = self
+
+      def health
+        worker = request(:health, timeout: @request_timeout)
+        worker = @worker_health.get unless worker.is_a?(Hash)
+        @worker_health.set(worker) if worker.is_a?(Hash)
+
+        @health.snapshot(
+          in_flight: @in_flight.value,
+          max_queue: @max_queue,
+          status: status_for(worker),
+          worker: worker
+        )
+      end
+
+      private
+
+      def validate_callable(callable, name:)
+        Core::Validation.validate_callable!(callable, name: name)
+        callable
+      end
+
+      def initialize_tracking
+        @closed = Concurrent::AtomicReference.new(false)
+        @health = Core::Integration::DestinationHealth.new(counter_keys: COUNTER_KEYS, failure_counter: nil)
+        @in_flight = Concurrent::AtomicFixnum.new(0)
+        @worker_health = Concurrent::AtomicReference.new
+      end
+
+      def start_worker
+        @ack_port = ::Ractor::Port.new
+        setup_port = ::Ractor::Port.new
+        @worker = spawn_worker(setup_port)
+        @port = receive_worker_port(setup_port)
+        @ack_thread = start_ack_thread
+      rescue StandardError => e
+        raise ArgumentError, "ractor destination collaborators must be ractor-copyable or shareable: #{e.message}"
+      ensure
+        PortLifecycle.close(setup_port) if defined?(setup_port) && setup_port
+      end
+
+      def spawn_worker(setup_port)
+        ack_port = @ack_port
+        formatter = @formatter
+        encoder = @encoder
+        output = @output
+        max_record_bytes = @max_record_bytes
+        close_output = @close_output
+
+        # :nocov:
+        ::Ractor.new(setup_port, ack_port, formatter, encoder, output, max_record_bytes, close_output,
+                     name: "julewire-ractor-destination") do |worker_port, worker_ack_port, worker_formatter,
+                                                             worker_encoder, worker_output, worker_max_record_bytes,
+                                                             worker_close_output|
+          command_port = ::Ractor::Port.new
+          worker_port.send(command_port)
+          DestinationWorker.run(
+            command_port: command_port,
+            ack_port: worker_ack_port,
+            formatter: worker_formatter,
+            encoder: worker_encoder,
+            output: worker_output,
+            max_record_bytes: worker_max_record_bytes,
+            close_output: worker_close_output
+          )
+        end
+        # :nocov:
+      end
+
+      def receive_worker_port(setup_port)
+        selected, value = ::Ractor.select(setup_port, @worker)
+        return value if selected.equal?(setup_port) && value.is_a?(::Ractor::Port)
+
+        raise ArgumentError, "ractor destination worker did not start"
+      end
+
+      def start_ack_thread
+        thread = Thread.new do
+          loop do
+            message = @ack_port.receive
+            break if message.is_a?(Hash) && message[:event] == :closed
+
+            handle_ack(message)
+          end
+        rescue StandardError => e
+          record_failure(e, phase: :ack)
+        end
+        thread.name = "julewire-ractor-destination-ack"
+        thread.report_on_exception = true
+        thread
+      end
+
+      def handle_ack(message)
+        return unless message.is_a?(Hash) && message[:event] == :ack
+
+        decrement_in_flight
+        case message[:status]
+        when :accepted
+          increment(:worker_accepted)
+        when :dropped
+          increment(:worker_dropped)
+        end
+      end
+
+      def request(command, timeout:)
+        return false if closed? && command != :health && command != :close
+
+        reply = ::Ractor::Port.new
+        @port.send({ command: command, reply: reply })
+        wait_for_reply(reply, timeout)
+      rescue StandardError => e
+        record_failure(e, phase: :request, command: command)
+        false
+      ensure
+        PortLifecycle.close(reply) if reply
+      end
+
+      def wait_for_reply(reply, timeout)
+        return reply.receive unless timeout
+
+        timeout_port = ::Ractor::Port.new
+        token = @scheduler.schedule(timeout) do
+          timeout_port.send(:timeout)
+        rescue StandardError
+          nil
+        end
+        selected, response = ::Ractor.select(reply, timeout_port)
+
+        selected.equal?(timeout_port) ? false : response
+      ensure
+        @scheduler.cancel(token) if defined?(token)
+        PortLifecycle.close(timeout_port) if defined?(timeout_port) && timeout_port
+      end
+
+      def queue_full?
+        @max_queue.positive? && @in_flight.value >= @max_queue
+      end
+
+      def closed? = @closed.get
+
+      def close_ports
+        @port&.send({ command: :close_worker }) unless @port&.closed?
+      rescue StandardError
+        nil
+      ensure
+        PortLifecycle.close(@port) if @port
+        PortLifecycle.close(@ack_port) if @ack_port
+      end
+
+      def drop(reason, record)
+        record_loss(reason, record)
+        Core::Diagnostics::CallbackNotifier.call(
+          @on_drop,
+          reason,
+          { destination: name, phase: :ractor_destination, reason: reason }
+        )
+        nil
+      end
+
+      def record_loss(reason, record)
+        metadata = Core::Records::Metadata.call(record)
+        @health.record_loss(
+          reason: reason,
+          event: metadata[:event],
+          severity: metadata[:severity],
+          source: metadata[:source]
+        )
+      end
+
+      def record_failure(error, **metadata)
+        @health.record_failure(error, counter: nil, **metadata)
+        Core::Diagnostics::CallbackNotifier.call(@on_failure, error, { destination: name }.merge(metadata))
+      rescue StandardError
+        nil
+      end
+
+      def decrement_in_flight
+        @in_flight.decrement if @in_flight.value.positive?
+      end
+
+      def increment(key)
+        @health.increment(key)
+      end
+
+      def status_for(worker)
+        return :closed if closed?
+        return :degraded if @health.degraded?
+        return :degraded if worker.is_a?(Hash) && worker[:status] == :degraded
+
+        :ok
+      end
+    end
+  end
+end

data/lib/julewire/ractor/destination_worker.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+# :nocov:
+module Julewire
+  module Ractor
+    class DestinationWorker
+      COUNTER_KEYS = %i[
+        encode_error
+        formatter_error
+        formatted
+        output_accepted
+        output_error
+        output_exception
+        output_rejected
+        received
+        record_too_large
+      ].freeze
+      private_constant :COUNTER_KEYS
+
+      class << self
+        def run(command_port:, ack_port:, formatter:, encoder:, output:, max_record_bytes:, close_output:)
+          new(formatter: formatter, encoder: encoder, output: output, max_record_bytes: max_record_bytes,
+              close_output: close_output).run(command_port: command_port, ack_port: ack_port)
+        end
+      end
+
+      def initialize(formatter:, encoder:, output:, max_record_bytes:, close_output:)
+        @formatter = formatter
+        @encoder = encoder
+        @output = output
+        @max_record_bytes = max_record_bytes
+        @close_output = close_output
+        @health = Core::Integration::DestinationHealth.new(counter_keys: COUNTER_KEYS, failure_counter: nil)
+        @write_step = Core::Destinations::WriteStep.new(
+          formatter: @formatter,
+          encoder: @encoder,
+          output: @output,
+          max_record_bytes: @max_record_bytes,
+          increment: method(:increment),
+          failure: method(:record_write_step_failure),
+          loss: method(:record_write_step_loss),
+          output_class_name: method(:output_class_name)
+        )
+      end
+
+      def run(command_port:, ack_port:)
+        @ack_port = ack_port
+        loop do
+          message = command_port.receive
+          break if close_message?(message)
+
+          break if dispatch(message) == :close
+        end
+      ensure
+        close_output
+        ack(:closed)
+      end
+
+      private
+
+      def close_message?(message)
+        message.is_a?(Hash) && message[:command] == :close_worker
+      end
+
+      def dispatch(message)
+        return unless message.is_a?(Hash)
+
+        case message[:command]
+        when :emit
+          emit(message[:record])
+        when :flush
+          reply_to(message, call_output_lifecycle(:flush))
+        when :close
+          reply_to(message, call_output_lifecycle(:close))
+          :close
+        when :health
+          reply_to(message, health)
+        end
+      rescue StandardError => e
+        record_failure(e, phase: :dispatch)
+        reply_to(message, false)
+      end
+
+      def emit(record)
+        ack(@write_step.call(record) == :accepted ? :accepted : :dropped)
+      rescue StandardError => e
+        record_failure(e, phase: :emit)
+        ack(:dropped)
+      end
+
+      def call_output_lifecycle(method_name)
+        return close_lifecycle if method_name == :close
+        return true unless @output.respond_to?(method_name)
+
+        @output.public_send(method_name) != false
+      rescue StandardError => e
+        record_failure(e, phase: :output_lifecycle, action: method_name)
+        false
+      end
+
+      def close_lifecycle
+        return true if output_closed?
+        return @output.close != false if @close_output && @output.respond_to?(:close)
+        return @output.flush != false if @output.respond_to?(:flush)
+
+        true
+      rescue StandardError => e
+        record_failure(e, phase: :output_lifecycle, action: :close)
+        false
+      end
+
+      def close_output
+        return if output_closed?
+        return unless @close_output && @output.respond_to?(:close)
+
+        @output.close
+      rescue StandardError => e
+        record_failure(e, phase: :output_lifecycle, action: :close)
+      end
+
+      def output_closed?
+        @output.respond_to?(:closed?) ? @output.closed? : false
+      end
+
+      def health
+        @health.snapshot
+      end
+
+      def increment(key)
+        @health.increment(key)
+      end
+
+      def record_failure(error, **metadata)
+        @health.record_failure(error, counter: nil, **metadata)
+      end
+
+      def record_loss(reason, **metadata)
+        @health.record_loss(reason: reason, counter: nil, **metadata)
+      end
+
+      def record_write_step_failure(error, metadata)
+        record_failure(error, **recordless_metadata(metadata))
+      end
+
+      def record_write_step_loss(reason, metadata)
+        return if %i[formatter_error encode_error].include?(reason)
+
+        record_loss(reason, **recordless_metadata(metadata))
+      end
+
+      def output_class_name
+        @output.class.name
+      end
+
+      def recordless_metadata(metadata)
+        metadata.except(:record)
+      end
+
+      def ack(status)
+        @ack_port.send({ event: :ack, status: status })
+      rescue StandardError
+        nil
+      end
+
+      def reply_to(message, response)
+        reply = message[:reply]
+        reply.send(response) if reply.is_a?(::Ractor::Port)
+      rescue StandardError => e
+        record_failure(e, phase: :reply)
+      end
+    end
+
+    private_constant :DestinationWorker
+  end
+end
+# :nocov:

data/lib/julewire/ractor/fanout.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Ractor
+    class Fanout
+      attr_reader :name
+
+      def initialize(destinations:, name: :ractor_fanout, on_failure: nil)
+        @name = Core::Destinations.normalize_name(name)
+        @destinations = Array(destinations).map { normalize_destination(it) }.freeze
+        raise ArgumentError, "destinations must not be empty" if @destinations.empty?
+
+        Core::Validation.validate_callable!(on_failure, name: :on_failure, allow_nil: true)
+        @on_failure = on_failure
+        @health = Core::Integration::DestinationHealth.new(counter_keys: [], failure_counter: nil)
+      end
+
+      def emit(record)
+        @destinations.each { emit_to_destination(it, record) }
+        nil
+      end
+
+      def flush(timeout: nil)
+        call_lifecycle(:flush, timeout: timeout)
+      end
+
+      def close(timeout: nil)
+        call_lifecycle(:close, timeout: timeout)
+      end
+
+      def after_fork!
+        @destinations.each do |destination|
+          destination.after_fork! if destination.respond_to?(:after_fork!)
+        rescue StandardError => e
+          record_failure(e, action: :after_fork, destination: destination.name)
+        end
+        self
+      end
+
+      def resource_identity = self
+
+      def health
+        destinations = @destinations.to_h { [it.name, destination_health(it)] }
+        @health.snapshot(
+          destinations: destinations,
+          status: health_status(destinations)
+        )
+      end
+
+      private
+
+      def normalize_destination(value)
+        destination = value.is_a?(Hash) ? Destination.new(**value) : value
+        Core::Destinations::Registry.validate!(destination)
+      end
+
+      def emit_to_destination(destination, record)
+        destination.emit(record)
+      rescue StandardError => e
+        record_failure(e, action: :emit, destination: destination.name, record_metadata: Core::Records::Metadata.call(record))
+      end
+
+      def call_lifecycle(method_name, timeout:)
+        ok = true
+        @destinations.each do |destination|
+          ok = false if destination.public_send(method_name, timeout: timeout) == false
+        rescue StandardError => e
+          record_failure(e, action: method_name, destination: destination.name)
+          ok = false
+        end
+        ok
+      end
+
+      def destination_health(destination)
+        destination.health
+      rescue StandardError => e
+        Core::Diagnostics::FailureSnapshot.build(e, destination: destination.name, phase: :ractor_fanout_health)
+      end
+
+      def health_status(destinations)
+        return :degraded if @health.last_failure
+        return :degraded if destinations.any? { |_name, health| health[:status] == :degraded || health[:phase] }
+
+        :ok
+      end
+
+      def record_failure(error, **metadata)
+        @health.record_failure(error, counter: nil, phase: :ractor_fanout, **metadata)
+        @on_failure&.call(error, **metadata, phase: :ractor_fanout)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/julewire/ractor/port_lifecycle.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Ractor
+    module PortLifecycle
+      class << self
+        def close(port)
+          return unless port.respond_to?(:close)
+          return if port.respond_to?(:closed?) && port.closed?
+
+          port.close
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/julewire/ractor/remote_payload.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Ractor
+    module RemotePayload
+      MISSING = Object.new.freeze
+      private_constant :MISSING
+
+      class << self
+        def extract(payload)
+          {
+            input: input_value(payload),
+            context: hash_value(payload, :context),
+            neutral: hash_value(payload, :neutral),
+            attributes: hash_value(payload, :attributes),
+            carry: hash_value(payload, :carry),
+            scope: scope_snapshot(hash_value(payload, :scope))
+          }
+        end
+
+        def input_value(payload)
+          value = Core::Integration::Values::Read.hash_value(payload, :input, default: MISSING)
+          value.equal?(MISSING) ? {} : value
+        end
+
+        def scope_snapshot(scope_payload)
+          Core::Execution::ScopeSnapshot.new(
+            execution: hash_value(scope_payload, :execution),
+            neutral: hash_value(scope_payload, :neutral),
+            attributes: hash_value(scope_payload, :attributes),
+            carry: hash_value(scope_payload, :carry),
+            labels: hash_value(scope_payload, :labels)
+          )
+        end
+
+        def hash_value(hash, key)
+          value = Core::Integration::Values::Read.hash_value(hash, key)
+          value.is_a?(Hash) ? Core::Fields::FieldSet.deep_symbolize_keys(value) : {}
+        end
+      end
+    end
+  end
+end