cdc-parallel 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 727373877c3c90e65e65ec4bbb5f5312b7e0d15d879af54402fa6752ac730aa0
-  data.tar.gz: cd2701bf5e93f5ef825f4b2e95a015dfbbb214bb14cb3f095a65d499aeaf9f28
+  metadata.gz: e6e397f5b9c12042cf5521d0cc689604560c0a4180a9a53911915ebc38d31e27
+  data.tar.gz: a7736d3c1958ca24183939c4221fa9812a558fab0daab814988d2ba225afc4aa
 SHA512:
-  metadata.gz: 33e6152dfaf7fdeda19853229519f2154b6ec39cb87b3c074dea0658e94eb65156d11d8e7ce11d0c3560dae678fe196dc1cc99a4f6fb4c6d712f8b84be7ad274
-  data.tar.gz: 9999fdf8a4e05694f506b1a4f6e6db9679bc0da09d47b8c67513959d8467ead0fce0b26245b158c3101c8467110ed29af2aefeb4bc13c27d95ac76e38920e449
+  metadata.gz: 75fbe128470e921d71ccb21f98969751379bf63a2b2fc8e1fae9fb45d5557ce281a025e6c67e4a02a3438300380671d76812535c8fe9b271225b7356605e8b6f
+  data.tar.gz: 46960241a8cdc582a26c63c6725e2bae4cce316c3206019e03c3123ed178b891206e30a883b2932e5984f8a03822b640dea6f7672ff54747d77fbc570438be3d

data/CHANGELOG.md

@@ -6,7 +6,18 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 
 ## Unreleased
 
-No unreleased changes.
+## [0.2.4] - 2026-06-28
+
+### Fixed
+
+- Started `Runtime` processors before internal pools shareablize the processor object graph.
+- Forwarded `Runtime` supervisor tuning options to both event and transaction pools.
+
+### Added
+
+- Added fatal-worker supervisor regression tests for respawn, degraded cooldown, and shutdown behavior.
+- Added runtime regression tests for lifecycle ordering and supervisor option forwarding.
+- Updated RBS signatures and supervisor plan documentation for the new runtime contract.
 
 ## [0.2.3] - 2026-06-03
 

data/lib/cdc/parallel/configuration.rb

@@ -2,6 +2,10 @@
 
 module CDC
   module Parallel
+    # :nodoc:
+    ConfigurationData = Data.define(:size, :timeout)
+    private_constant :ConfigurationData
+
     # Immutable configuration shared by cdc-parallel runtime objects.
     #
     # `Configuration` validates worker sizing and timeout values at construction
@@ -22,7 +26,7 @@ module CDC
     # @!attribute [r] timeout
     #   @return [Numeric, nil] Optional wait timeout in seconds.
     # @api public
-    class Configuration < Data.define(:size, :timeout)
+    class Configuration < ConfigurationData
       # Create a validated runtime configuration.
       #
       # @param size [Integer]

data/lib/cdc/parallel/processor_pool.rb

@@ -109,22 +109,70 @@ module CDC
       # @raise [ArgumentError]
       #   Raised by {Configuration} when `size` or `timeout` is invalid.
       # @return [void]
-      def initialize(processor:, size: Etc.nprocessors, timeout: nil)
+      # @param supervision [Boolean]
+      #   Whether worker Ractors should be respawned after unexpected death.
+      # @param max_respawns [Integer]
+      #   Maximum crash count inside `respawn_window` before a worker slot enters
+      #   cooldown.
+      # @param respawn_window [Numeric]
+      #   Time window, in seconds, used by the crash-loop circuit breaker.
+      # @param respawn_cooldown [Numeric]
+      #   Cooldown, in seconds, before a crash-looping slot is revived again.
+      # @param manage_lifecycle [Boolean]
+      #   When `true` (default), the pool calls `processor.start` during
+      #   initialization and `processor.flush` + `processor.stop` during shutdown.
+      #   Set to `false` when a higher-level runtime (e.g. {Runtime}) owns the
+      #   processor lifecycle so that `start`/`stop`/`flush` are not called
+      #   multiple times when the same processor is shared across pools.
+      # rubocop:disable Metrics/MethodLength
+      def initialize(
+        processor:,
+        size: Etc.nprocessors,
+        timeout: nil,
+        supervision: true,
+        max_respawns: 3,
+        respawn_window: 60,
+        respawn_cooldown: 5,
+        manage_lifecycle: true
+      )
         validate_processor!(processor)
 
+        processor.start if manage_lifecycle
         @processor = ::Ractor.make_shareable(processor)
+        @manage_lifecycle = manage_lifecycle
         @configuration = Configuration.new(size:, timeout:)
-        booted_workers = Array.new(@configuration.size) do
-          build_worker(@processor)
-        end
-
-        @workers = booted_workers.map(&:first).freeze
-        @worker_inboxes = booted_workers.map(&:last).freeze
+        @slots = Array.new(@configuration.size) do |index|
+          WorkerSlot.new(
+            index:,
+            processor: @processor,
+            supervision:,
+            max_respawns:,
+            respawn_window:,
+            respawn_cooldown:
+          )
+        end.freeze
+        @workers = @slots.map(&:worker).freeze
+        @worker_inboxes = @slots.map(&:inbox).freeze
 
         @next_worker = 0
         @dispatch_mutex = Mutex.new
         @shutdown = false
       end
+      # rubocop:enable Metrics/MethodLength
+
+      # Return total worker-slot respawns since this pool booted.
+      #
+      # @return [Integer]
+      def respawns
+        @slots.sum(&:respawns)
+      end
+
+      # Return whether any worker slot is currently in crash-loop cooldown.
+      #
+      # @return [Boolean]
+      def degraded?
+        @slots.any?(&:degraded?)
+      end
 
       # Process one work item synchronously.
       #
@@ -161,9 +209,9 @@ module CDC
         work_items = items.map { |item| ::Ractor.make_shareable(item) }
         reply_port = ::Ractor::Port.new
 
-        dispatch(work_items, reply_port)
+        assignments = dispatch(work_items, reply_port)
 
-        collect_results(reply_port, work_items.length)
+        collect_results(reply_port, work_items.length, assignments).compact.freeze
       ensure
         reply_port&.close
       end
@@ -184,6 +232,10 @@ module CDC
         end
 
         wait_for_workers
+        return unless @manage_lifecycle
+
+        @processor.flush
+        @processor.stop
       end
 
       private
@@ -192,25 +244,25 @@ module CDC
         @dispatch_mutex.synchronize do
           raise ShutdownError, "processor pool has been shut down" if @shutdown
 
+          assignments = Array.new(work_items.length)
           work_items.each_with_index do |item, index|
-            next_worker_inbox.send([index, item, reply_port])
+            slot = next_worker_slot
+            assignments[index] = slot
+            slot.send_work(index, item, reply_port)
           end
+          assignments
         end
       end
 
       def signal_workers
-        @worker_inboxes.each do |inbox|
-          inbox.send(nil)
-        rescue Ractor::ClosedError
-          # Already stopped.
-        end
+        @slots.each(&:shutdown)
       end
 
       def wait_for_workers
         if @configuration.timeout
           wait_for_workers_with_timeout
         else
-          @workers.each(&:join)
+          @slots.each(&:join)
         end
       end
 
@@ -220,11 +272,11 @@ module CDC
 
         deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
 
-        @workers.each do |worker|
+        @slots.each do |slot|
           remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
           break unless remaining.positive?
 
-          ::Timeout.timeout(remaining, TimeoutError) { worker.join }
+          ::Timeout.timeout(remaining, TimeoutError) { slot.join }
         rescue TimeoutError
           break
         end
@@ -238,16 +290,7 @@ module CDC
               "#{processor.class} must declare ractor_safe!"
       end
 
-      def build_worker(processor)
-        boot_port = ::Ractor::Port.new
-        worker = start_worker(processor, boot_port)
-
-        [worker, boot_port.receive]
-      ensure
-        boot_port.close
-      end
-
-      def start_worker(processor, boot_port)
+      def self.start_worker(processor, boot_port)
         ::Ractor.new(processor, boot_port) do |safe_processor, ready_port|
           inbox = ::Ractor::Port.new
           ready_port << inbox
@@ -255,6 +298,7 @@ module CDC
           CDC::Parallel::ProcessorPool.send(:run_worker_loop, safe_processor, inbox)
         end
       end
+      private_class_method :start_worker
 
       def self.run_worker_loop(safe_processor, inbox)
         loop do
@@ -282,36 +326,37 @@ module CDC
       end
       private_class_method :worker_response
 
-      def next_worker_inbox
-        inbox = @worker_inboxes[@next_worker]
+      def next_worker_slot
+        slot = @slots[@next_worker]
 
         @next_worker += 1
-        @next_worker = 0 if @next_worker >= @worker_inboxes.length
+        @next_worker = 0 if @next_worker >= @slots.length
 
-        inbox
+        slot
       end
 
-      def collect_results(reply_port, count)
+      def collect_results(reply_port, count, assignments = [])
         results = Array.new(count)
         return results.freeze if count.zero?
 
         if @configuration.timeout
-          collect_results_with_timeout(reply_port, results)
+          collect_results_with_timeout(reply_port, results, assignments)
         else
-          collect_results_without_timeout(reply_port, results)
+          collect_results_without_timeout(reply_port, results, assignments)
         end
       end
 
-      def collect_results_without_timeout(reply_port, results)
+      def collect_results_without_timeout(reply_port, results, assignments = [])
         results.length.times do
           index, response = reply_port.receive
           results[index] = ResultCollector.normalize(response)
+          assignments[index]&.complete(index)
         end
 
         results.freeze
       end
 
-      def collect_results_with_timeout(reply_port, results)
+      def collect_results_with_timeout(reply_port, results, assignments = [])
         timeout = @configuration.timeout
         return results.freeze unless timeout
 
@@ -323,6 +368,7 @@ module CDC
 
           index, response = ::Timeout.timeout(remaining, TimeoutError) { reply_port.receive }
           results[index] = ResultCollector.normalize(response)
+          assignments[index]&.complete(index)
         rescue TimeoutError
           return timeout_results(results)
         end
@@ -340,6 +386,271 @@ module CDC
           result || CDC::Core::ProcessorResult.failure(timeout_error)
         end.freeze
       end
+
+      # One supervised worker position in the pool.
+      #
+      # A slot keeps its identity while its underlying Ractor can be replaced
+      # after unexpected death. The slot also owns in-flight reply ports so a
+      # fatal worker exit can be reported back to callers instead of leaving
+      # them blocked forever.
+      #
+      # @api private
+      # rubocop:disable Metrics/ClassLength
+      class WorkerSlot
+        # Number of times this slot has booted a replacement worker.
+        #
+        # @return [Integer]
+        # @api private
+        attr_reader :respawns
+
+        # Create a supervised worker slot.
+        #
+        # @param index [Integer]
+        #   Stable slot index inside the owning pool.
+        # @param processor [CDC::Core::Processor]
+        #   Shareable processor instance used by the worker.
+        # @param supervision [Boolean]
+        #   Whether unexpected worker death should trigger respawn.
+        # @param max_respawns [Integer]
+        #   Maximum crash count inside the respawn window before cooldown.
+        # @param respawn_window [Numeric]
+        #   Sliding crash-loop window in seconds.
+        # @param respawn_cooldown [Numeric]
+        #   Cooldown duration in seconds after repeated crashes.
+        # @return [void]
+        # @api private
+        def initialize(index:, processor:, supervision:, max_respawns:, respawn_window:, respawn_cooldown:)
+          @index = index
+          @processor = processor
+          @supervision = supervision
+          @max_respawns = Integer(max_respawns)
+          @respawn_window = Float(respawn_window)
+          @respawn_cooldown = Float(respawn_cooldown)
+          @lock = Mutex.new
+          @inflight = {}
+          @crashes = []
+          @respawns = 0
+          @shutdown = false
+          @degraded_until = nil
+          boot!
+          @supervisor_thread = Thread.new { supervise }
+        end
+
+        # Return the current worker Ractor.
+        #
+        # @return [Ractor]
+        # @api private
+        def worker
+          @lock.synchronize { @worker }
+        end
+
+        # Return the current worker inbox port.
+        #
+        # @return [Ractor::Port]
+        # @api private
+        def inbox
+          @lock.synchronize { @inbox }
+        end
+
+        # Send one indexed work item to the current worker.
+        #
+        # If the slot is cooling down or the worker inbox is already closed, a
+        # serialized failure response is sent to the caller reply port.
+        #
+        # @param index [Integer]
+        # @param item [Object]
+        # @param reply_port [Ractor::Port]
+        # @return [void]
+        # @api private
+        def send_work(index, item, reply_port)
+          target = nil
+          immediate_failure = nil
+
+          @lock.synchronize do
+            if degraded_locked?
+              immediate_failure = RuntimeError.new("worker slot #{@index} is cooling down after repeated crashes")
+            else
+              @inflight[index] = reply_port
+              target = @inbox
+            end
+          end
+
+          return send_failure(reply_port, index, require_failure(immediate_failure)) if immediate_failure
+
+          send_to_inbox(target, [index, item, reply_port])
+        rescue Ractor::ClosedError => e
+          complete(index)
+          send_failure(reply_port, index, e)
+        end
+
+        # Mark an in-flight work item as completed.
+        #
+        # @param index [Integer]
+        # @return [void]
+        # @api private
+        def complete(index)
+          @lock.synchronize { @inflight.delete(index) }
+        end
+
+        # Return whether this slot is currently in crash-loop cooldown.
+        #
+        # @return [Boolean]
+        # @api private
+        def degraded?
+          @lock.synchronize { degraded_locked? }
+        end
+
+        # Return whether this slot is degraded while the caller holds the lock.
+        #
+        # @return [Boolean]
+        # @api private
+        def degraded_locked?
+          degraded_until = @degraded_until
+          return false unless degraded_until
+
+          if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= degraded_until
+            @degraded_until = nil
+            false
+          else
+            true
+          end
+        end
+
+        # Request worker shutdown and prevent future respawns.
+        #
+        # @return [void]
+        # @api private
+        def shutdown
+          target = nil
+          @lock.synchronize do
+            @shutdown = true
+            target = @inbox
+          end
+
+          send_to_inbox(target, nil)
+        rescue Ractor::ClosedError
+          nil
+        end
+
+        # Return a non-nil failure object for Steep and runtime dispatch.
+        #
+        # @param failure [Exception, nil]
+        # @return [Exception]
+        # @raise [RuntimeError] if no failure was supplied
+        # @api private
+        def require_failure(failure)
+          raise "worker failure unavailable" if failure.nil?
+
+          failure
+        end
+
+        # Send a message to a worker inbox.
+        #
+        # Ractor::Port supports both #send and #<<. Some tests replace the inbox
+        # with a small sentinel object that only implements #send so the closed
+        # inbox path can be exercised without relying on a live port. Keep this
+        # helper deliberately typed as an inbox boundary rather than forcing every
+        # caller to narrow the test double shape.
+        #
+        # @param inbox [#send, nil]
+        # @param message [Object]
+        # @return [void]
+        # @raise [Ractor::ClosedError] when the worker inbox has already closed
+        # @api private
+        def send_to_inbox(inbox, message)
+          raise Ractor::ClosedError, "worker inbox closed" if inbox.nil?
+
+          inbox.send(message)
+          nil
+        end
+
+        # Wait for the supervisor thread to finish.
+        #
+        # @return [Thread]
+        # @api private
+        def join
+          @supervisor_thread.join
+        end
+
+        private
+
+        def boot!
+          boot_port = ::Ractor::Port.new
+          worker = CDC::Parallel::ProcessorPool.send(:start_worker, @processor, boot_port)
+          inbox = boot_port.receive
+
+          @lock.synchronize do
+            @worker = worker
+            @inbox = inbox
+          end
+        ensure
+          boot_port&.close
+        end
+
+        def supervise
+          loop do
+            current_worker = worker
+            cause = wait_for_worker_death(current_worker)
+            break if shutting_down?
+
+            fail_inflight(cause)
+            break unless @supervision
+
+            cool_down_if_needed
+            break if shutting_down?
+
+            boot!
+            @lock.synchronize { @respawns += 1 }
+          end
+        end
+
+        def wait_for_worker_death(current_worker)
+          current_worker.value
+          RuntimeError.new("worker slot #{@index} exited unexpectedly")
+        rescue Ractor::Error => e
+          cause = e.respond_to?(:cause) ? e.cause : nil
+          cause.is_a?(Exception) ? cause : e
+        end
+
+        def fail_inflight(cause)
+          pending = {} # : Hash[Integer, Ractor::Port]
+          @lock.synchronize do
+            pending = @inflight.dup
+            @inflight.clear
+          end
+
+          pending.each { |index, reply_port| send_failure(reply_port, index, cause) }
+        end
+
+        def send_failure(reply_port, index, error)
+          reply_port << [index, ResultCollector.worker_failure(error)]
+        rescue Ractor::ClosedError
+          nil
+        end
+
+        def cool_down_if_needed
+          return unless crash_loop?
+
+          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @respawn_cooldown
+          @lock.synchronize { @degraded_until = deadline }
+          sleep @respawn_cooldown
+          @lock.synchronize { @degraded_until = nil }
+        end
+
+        def crash_loop?
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          @lock.synchronize do
+            @crashes = @crashes.select { |timestamp| now - timestamp <= @respawn_window }
+            @crashes << now
+            @crashes.length > @max_respawns
+          end
+        end
+
+        def shutting_down?
+          @lock.synchronize { @shutdown }
+        end
+      end
+      # rubocop:enable Metrics/ClassLength
     end
   end
 end

data/lib/cdc/parallel/result_collector.rb

@@ -90,6 +90,12 @@ module CDC
         end
       end
 
+      # Return whether a value is a serialized worker failure payload.
+      #
+      # @param value [Object]
+      #   Raw worker response.
+      # @return [Boolean]
+      # @api private
       def self.worker_failure?(value)
         value.is_a?(Hash) && value[:type] == FAILURE_MARKER
       end

data/lib/cdc/parallel/runtime.rb

@@ -48,14 +48,39 @@ module CDC
       #   Number of worker Ractors per internal pool.
       # @param timeout [Numeric, nil]
       #   Optional timeout in seconds for result collection and shutdown waits.
+      # @param supervision [Boolean]
+      #   Whether worker Ractors should be respawned after unexpected death.
+      # @param max_respawns [Integer]
+      #   Maximum crash count inside `respawn_window` before a worker slot enters
+      #   cooldown.
+      # @param respawn_window [Numeric]
+      #   Time window, in seconds, used by the crash-loop circuit breaker.
+      # @param respawn_cooldown [Numeric]
+      #   Cooldown, in seconds, before a crash-looping slot is revived again.
       # @raise [UnsafeProcessorError]
       #   Raised when the processor class has not declared `ractor_safe!`.
       # @raise [ArgumentError]
       #   Raised when size or timeout is invalid.
       # @return [void]
-      def initialize(processor:, size: Etc.nprocessors, timeout: nil)
-        @processor_pool = ProcessorPool.new(processor:, size:, timeout:)
-        @transaction_pool = TransactionPool.new(processor:, size:, timeout:)
+      def initialize(
+        processor:,
+        size: Etc.nprocessors,
+        timeout: nil,
+        supervision: true,
+        max_respawns: 3,
+        respawn_window: 60,
+        respawn_cooldown: 5
+      )
+        @processor = processor
+        @processor.start
+
+        pool_options = build_pool_options(
+          processor:, size:, timeout:, supervision:,
+          max_respawns:, respawn_window:, respawn_cooldown:
+        )
+
+        @processor_pool = ProcessorPool.new(**pool_options)
+        @transaction_pool = TransactionPool.new(**pool_options)
         @router = Router.new(processor_pool: @processor_pool, transaction_pool: @transaction_pool)
         @shutdown = false
       end
@@ -94,7 +119,9 @@ module CDC
       # Shut down all runtime resources.
       #
       # Shutdown is idempotent and cascades to the internal event and transaction
-      # pools. After shutdown, {#process} raises {ShutdownError}.
+      # pools. The processor's `flush` and `stop` lifecycle hooks are called once
+      # after all pools have been drained. After shutdown, {#process} raises
+      # {ShutdownError}.
       #
       # @return [void]
       def shutdown
@@ -103,6 +130,46 @@ module CDC
         @shutdown = true
         @processor_pool.shutdown
         @transaction_pool.shutdown
+        @processor.flush
+        @processor.stop
+      end
+
+      private
+
+      # Build keyword arguments shared by the event and transaction pools.
+      #
+      # The runtime owns the processor lifecycle, so internal pools receive
+      # `manage_lifecycle: false` to avoid duplicate `start`, `flush`, or `stop`
+      # calls for the same processor instance.
+      #
+      # @param processor [CDC::Core::Processor]
+      # @param size [Integer]
+      # @param timeout [Numeric, nil]
+      # @param supervision [Boolean]
+      # @param max_respawns [Integer]
+      # @param respawn_window [Numeric]
+      # @param respawn_cooldown [Numeric]
+      # @return [Hash]
+      # @api private
+      def build_pool_options(
+        processor:,
+        size:,
+        timeout:,
+        supervision:,
+        max_respawns:,
+        respawn_window:,
+        respawn_cooldown:
+      )
+        {
+          processor:,
+          size:,
+          timeout:,
+          supervision:,
+          max_respawns:,
+          respawn_window:,
+          respawn_cooldown:,
+          manage_lifecycle: false
+        }
       end
     end
   end

data/lib/cdc/parallel/transaction_pool.rb

@@ -38,11 +38,42 @@ module CDC
       #   Number of worker Ractors in the underlying processor pool.
       # @param timeout [Numeric, nil]
       #   Optional timeout in seconds for result collection and shutdown waits.
+      # @param supervision [Boolean]
+      #   Whether worker Ractors should be respawned after unexpected death.
+      # @param max_respawns [Integer]
+      #   Maximum crash count inside `respawn_window` before a worker slot enters
+      #   cooldown.
+      # @param respawn_window [Numeric]
+      #   Time window, in seconds, used by the crash-loop circuit breaker.
+      # @param respawn_cooldown [Numeric]
+      #   Cooldown, in seconds, before a crash-looping slot is revived again.
+      # @param manage_lifecycle [Boolean]
+      #   When `true` (default), the pool calls `processor.start` during
+      #   initialization and `processor.flush` + `processor.stop` during shutdown.
+      #   Set to `false` when a higher-level runtime owns the processor lifecycle.
       # @raise [UnsafeProcessorError]
       #   Raised when the processor class has not declared `ractor_safe!`.
       # @return [void]
-      def initialize(processor:, size: Etc.nprocessors, timeout: nil)
-        @processor_pool = ProcessorPool.new(processor:, size:, timeout:)
+      def initialize(
+        processor:,
+        size: Etc.nprocessors,
+        timeout: nil,
+        supervision: true,
+        max_respawns: 3,
+        respawn_window: 60,
+        respawn_cooldown: 5,
+        manage_lifecycle: true
+      )
+        @processor_pool = ProcessorPool.new(
+          processor:,
+          size:,
+          timeout:,
+          supervision:,
+          max_respawns:,
+          respawn_window:,
+          respawn_cooldown:,
+          manage_lifecycle:
+        )
       end
 
       # Process all events inside a transaction envelope.

data/lib/cdc/parallel/version.rb

@@ -8,6 +8,6 @@ module CDC
     # the loaded runtime version at boot time.
     #
     # @return [String]
-    VERSION = "0.2.3"
+    VERSION = "0.2.4"
   end
 end

data/lib/cdc/parallel.rb

@@ -12,6 +12,12 @@ require_relative "parallel/transaction_pool"
 require_relative "parallel/router"
 require_relative "parallel/runtime"
 
+# Namespace for Change Data Capture ecosystem components.
+#
+# The `cdc-parallel` gem adds the {CDC::Parallel} runtime namespace under
+# this shared ecosystem root.
+#
+# @api public
 module CDC
   # Optional parallel Change Data Capture runtime for `cdc-core` processors.
   #

data/sig/cdc/parallel/configuration.rbs

@@ -1,7 +1,13 @@
 module CDC
   module Parallel
+    # Private Data superclass used to keep YARD documentation stable for Configuration.
+    class ConfigurationData < Data
+      attr_reader size: Integer
+      attr_reader timeout: Numeric?
+    end
+
     # Immutable configuration for Ractor runtimes.
-    class Configuration < Data
+    class Configuration < ConfigurationData
       attr_reader size: Integer
 
       attr_reader timeout: Numeric?

data/sig/cdc/parallel/processor_pool.rbs

@@ -7,25 +7,46 @@ module CDC
       type worker_response = result | Hash[Symbol, untyped] | untyped
       type reply_port = Ractor::Port
       type worker_inbox = Ractor::Port
-      @processor: CDC::Core::Processor
 
+      @processor: CDC::Core::Processor
       @configuration: Configuration
-
+      @slots: Array[WorkerSlot]
       @workers: Array[Ractor]
-
       @worker_inboxes: Array[worker_inbox]
-
       @next_worker: Integer
-
       @dispatch_mutex: Mutex
-
       @shutdown: bool
+      @manage_lifecycle: bool
 
       # @param processor [CDC::Core::Processor]
       # @param size [Integer]
-      # @param timeout [Float, nil]
+      # @param timeout [Numeric, nil]
+      # @param supervision [Boolean]
+      # @param max_respawns [Integer]
+      # @param respawn_window [Numeric]
+      # @param respawn_cooldown [Numeric]
+      # @param manage_lifecycle [Boolean]
       # @return [void]
-      def initialize: (processor: CDC::Core::Processor, ?size: Integer, ?timeout: Numeric?) -> void
+      def initialize: (
+        processor: CDC::Core::Processor,
+        ?size: Integer,
+        ?timeout: Numeric?,
+        ?supervision: bool,
+        ?max_respawns: Integer,
+        ?respawn_window: Numeric,
+        ?respawn_cooldown: Numeric,
+        ?manage_lifecycle: bool
+      ) -> void
+
+      # Total worker-slot respawns since pool boot.
+      #
+      # @return [Integer]
+      def respawns: () -> Integer
+
+      # Whether any worker slot is in crash-loop cooldown.
+      #
+      # @return [Boolean]
+      def degraded?: () -> bool
 
       # Process one work item synchronously.
       #
@@ -39,40 +60,90 @@ module CDC
       # @return [Array<CDC::Core::ProcessorResult>]
       def process_many: (Array[work_item] items) -> Array[result]
 
-      # Shut down the pool.
+      # Shut down the pool and call processor lifecycle hooks if manage_lifecycle is true.
       #
       # @return [void]
-      def shutdown: () -> nil?
+      def shutdown: () -> void
 
       private
 
+      def self.start_worker: (CDC::Core::Processor processor, Ractor::Port boot_port) -> Ractor
+
       def self.run_worker_loop: (untyped safe_processor, untyped inbox) -> nil
 
       def self.worker_response: (untyped safe_processor, work_item item) -> worker_response
 
-      def dispatch: (Array[work_item] work_items, reply_port reply_port) -> Array[work_item]
+      def dispatch: (Array[work_item] work_items, reply_port reply_port) -> Array[WorkerSlot?]
 
-      def signal_workers: () -> Array[worker_inbox]
+      def signal_workers: () -> void
 
-      def wait_for_workers: () -> untyped
+      def wait_for_workers: () -> void
 
-      def wait_for_workers_with_timeout: () -> untyped
+      def wait_for_workers_with_timeout: () -> void
 
-      def validate_processor!: (CDC::Core::Processor processor) -> nil
+      def validate_processor!: (CDC::Core::Processor processor) -> void
 
-      def build_worker: (CDC::Core::Processor processor) -> ([Ractor, worker_inbox] | Array[untyped])
+      def next_worker_slot: () -> WorkerSlot
 
-      def start_worker: (CDC::Core::Processor processor, Ractor::Port boot_port) -> Ractor
+      def collect_results: (reply_port reply_port, Integer count, ?Array[WorkerSlot?] assignments) -> Array[result?]
 
-      def next_worker_inbox: () -> worker_inbox
+      def collect_results_without_timeout: (reply_port reply_port, Array[result?] results, Array[WorkerSlot?] assignments) -> Array[result?]
 
-      def collect_results: (reply_port reply_port, Integer count) -> Array[result]
-
-      def collect_results_without_timeout: (reply_port reply_port, Array[result?] results) -> untyped
-
-      def collect_results_with_timeout: (reply_port reply_port, Array[result?] results) -> untyped
+      def collect_results_with_timeout: (reply_port reply_port, Array[result?] results, Array[WorkerSlot?] assignments) -> Array[result?]
 
       def timeout_results: (Array[result?] results) -> Array[result]
+
+      # One supervised worker position in the pool.
+      class WorkerSlot
+        @index: Integer
+        @processor: CDC::Core::Processor
+        @supervision: bool
+        @max_respawns: Integer
+        @respawn_window: Float
+        @respawn_cooldown: Float
+        @lock: Mutex
+        @worker: Ractor
+        @inbox: Ractor::Port
+        @inflight: Hash[Integer, Ractor::Port]
+        @crashes: Array[Float]
+        @respawns: Integer
+        @shutdown: bool
+        @degraded_until: Float?
+        @supervisor_thread: Thread
+
+        attr_reader respawns: Integer
+
+        def initialize: (
+          index: Integer,
+          processor: CDC::Core::Processor,
+          supervision: bool,
+          max_respawns: Integer,
+          respawn_window: Numeric,
+          respawn_cooldown: Numeric
+        ) -> void
+
+        def worker: () -> Ractor
+        def inbox: () -> Ractor::Port
+        def send_work: (Integer index, untyped item, Ractor::Port reply_port) -> void
+        def complete: (Integer index) -> void
+        def degraded?: () -> bool
+        def shutdown: () -> void
+        def join: () -> void
+
+        private
+
+        def boot!: () -> void
+        def supervise: () -> void
+        def wait_for_worker_death: (Ractor current_worker) -> Exception
+        def fail_inflight: (Exception cause) -> void
+        def send_failure: (Ractor::Port reply_port, Integer index, Exception error) -> void
+        def require_failure: (Exception? failure) -> Exception
+        def send_to_inbox: (untyped inbox, untyped message) -> void
+        def cool_down_if_needed: () -> void
+        def crash_loop?: () -> bool
+        def shutting_down?: () -> bool
+        def degraded_locked?: () -> bool
+      end
     end
   end
 end

data/sig/cdc/parallel/runtime.rbs

@@ -4,19 +4,25 @@ module CDC
     class Runtime
       type work_item = CDC::Core::ChangeEvent | CDC::Core::TransactionEnvelope
 
+      @processor: CDC::Core::Processor
       @processor_pool: ProcessorPool
-
       @transaction_pool: TransactionPool
-
       @router: Router
-
       @shutdown: bool
 
       # @param processor [CDC::Core::Processor]
       # @param size [Integer]
-      # @param timeout [Float, nil]
+      # @param timeout [Numeric, nil]
       # @return [void]
-      def initialize: (processor: CDC::Core::Processor, ?size: Integer, ?timeout: Numeric?) -> void
+      def initialize: (
+        processor: CDC::Core::Processor,
+        ?size: Integer,
+        ?timeout: Numeric?,
+        ?supervision: bool,
+        ?max_respawns: Integer,
+        ?respawn_window: Numeric,
+        ?respawn_cooldown: Numeric
+      ) -> void
 
       # Process a ChangeEvent or TransactionEnvelope.
       #
@@ -31,10 +37,32 @@ module CDC
       # @return [CDC::Core::ProcessorResult]
       def process_transaction: (CDC::Core::TransactionEnvelope transaction) -> CDC::Core::ProcessorResult
 
-      # Shut down all runtime resources.
+      # Shut down all runtime resources and call processor lifecycle hooks.
       #
       # @return [void]
-      def shutdown: () -> nil
+      def shutdown: () -> void
+
+      private
+
+      # Build keyword arguments shared by internal pools.
+      def build_pool_options: (
+        processor: CDC::Core::Processor,
+        size: Integer,
+        timeout: Numeric?,
+        supervision: bool,
+        max_respawns: Integer,
+        respawn_window: Numeric,
+        respawn_cooldown: Numeric
+      ) -> {
+        processor: CDC::Core::Processor,
+        size: Integer,
+        timeout: Numeric?,
+        supervision: bool,
+        max_respawns: Integer,
+        respawn_window: Numeric,
+        respawn_cooldown: Numeric,
+        manage_lifecycle: false
+      }
     end
   end
 end

data/sig/cdc/parallel/transaction_pool.rbs

@@ -6,8 +6,18 @@ module CDC
 
       # @param processor [CDC::Core::Processor]
       # @param size [Integer]
-      # @param timeout [Float, nil]
-      def initialize: (processor: CDC::Core::Processor, ?size: Integer, ?timeout: Numeric?) -> void
+      # @param timeout [Numeric, nil]
+      # @param manage_lifecycle [Boolean]
+      def initialize: (
+        processor: CDC::Core::Processor,
+        ?size: Integer,
+        ?timeout: Numeric?,
+        ?supervision: bool,
+        ?max_respawns: Integer,
+        ?respawn_window: Numeric,
+        ?respawn_cooldown: Numeric,
+        ?manage_lifecycle: bool
+      ) -> void
 
       # Process all events inside a transaction envelope.
       #
@@ -18,7 +28,7 @@ module CDC
       # Shut down worker resources.
       #
       # @return [void]
-      def shutdown: () -> nil
+      def shutdown: () -> void
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cdc-parallel
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Ken C. Demanawa