cdc-parallel 0.2.1 → 0.2.3

data/lib/cdc/parallel/processor_pool.rb

@@ -2,75 +2,233 @@
 
 module CDC
   module Parallel
-    # Executes one Ractor-safe processor in pre-warmed persistent Ractor workers.
+    # Executes one Ractor-safe `cdc-core` processor across a fixed set of
+    # pre-warmed Ractor workers.
     #
-    # Workers are created during initialization and reused for every dispatch.
-    # This pays Ractor startup cost once, keeps workers alive after processor
-    # failures, and provides both synchronous single-item processing and batched
-    # dispatch for throughput-oriented benchmarks and runtimes.
+    # `ProcessorPool` is the low-level execution primitive used by
+    # {CDC::Parallel::Runtime}. It accepts normalized `cdc-core` work items,
+    # sends them across Ractor boundaries, invokes the configured processor, and
+    # returns `CDC::Core::ProcessorResult` objects in input order.
+    #
+    # This class is intentionally focused on **CPU-bound parallel execution**.
+    # Use it when the processor spends most of its time doing Ruby work such as
+    # transformation, enrichment, serialization, compression, scoring, or other
+    # in-memory computation. For I/O-heavy work, the CDC Ecosystem boundary is a
+    # future fiber-friendly runtime such as `cdc-concurrent`.
+    #
+    # ## Processor safety contract
+    #
+    # The supplied processor must declare `ractor_safe!` on its class. That
+    # declaration is treated as the processor author's explicit promise that the
+    # processor object and its dependencies can safely cross a Ractor boundary.
+    #
+    # `ProcessorPool` validates this declaration before booting workers:
+    #
+    # @example Declaring a processor as Ractor-safe
+    #   class AnalyticsProcessor < CDC::Core::Processor
+    #     ractor_safe!
+    #
+    #     def process(event)
+    #       CDC::Core::ProcessorResult.success(event)
+    #     end
+    #   end
+    #
+    #   pool = CDC::Parallel::ProcessorPool.new(
+    #     processor: AnalyticsProcessor.new,
+    #     size: 4
+    #   )
+    #
+    # Declaring `ractor_safe!` does not make unsafe code safe. It only allows the
+    # processor to be passed into worker Ractors. Mutable global state, database
+    # connections, sockets, caches, file handles, and non-shareable objects still
+    # need to be designed carefully by the processor implementor.
+    #
+    # ## Execution model
+    #
+    # Workers are created during initialization and reused for all dispatches.
+    # This pays Ractor startup cost once and keeps the pool stable even when
+    # individual processor calls fail.
+    #
+    # The pool uses a fan-out / fan-in pattern:
+    #
+    # ```text
+    # work items
+    #      |
+    #      v
+    # ProcessorPool
+    #      |
+    #      +----> Worker Ractor 1
+    #      +----> Worker Ractor 2
+    #      +----> Worker Ractor N
+    #                    |
+    #                    v
+    #             ProcessorResult
+    #                    |
+    #                    v
+    #              ordered results
+    # ```
+    #
+    # Fan-out uses round-robin worker selection. Fan-in collects responses from a
+    # reply port and reorders them by submission index, so `process_many` always
+    # returns results in the same order as the input array even when work
+    # completes out of order.
+    #
+    # @example Processing one item
+    #   result = pool.process(event)
+    #   result.success? #=> true
+    #
+    # @example Processing a batch while preserving result order
+    #   results = pool.process_many([event_a, event_b, event_c])
+    #   results.map(&:success?)
+    #
+    # @example Shutting down explicitly
+    #   pool.shutdown
+    #
+    # @note `ProcessorPool` preserves the order of returned results, not the
+    #   order in which independent items execute. If a sink needs strict ordering
+    #   by transaction, relation, or primary key, use the ecosystem ordering
+    #   contract and an ordered dispatcher/runtime above this primitive.
+    #
+    # @see CDC::Parallel::Runtime High-level facade for processing supported CDC work items
+    # @see CDC::Parallel::TransactionPool Transaction-envelope processing wrapper
+    # @see CDC::Parallel::ResultCollector Worker response normalization
+    # @api public
     class ProcessorPool # rubocop:disable Metrics/ClassLength
+      # Create a new pool and boot its worker Ractors.
+      #
       # @param processor [CDC::Core::Processor]
+      #   Processor instance used by every worker. Its class must respond to
+      #   `ractor_safe?` and return `true`.
       # @param size [Integer]
-      # @param timeout [Float, nil]
+      #   Number of worker Ractors to boot. Defaults to `Etc.nprocessors`.
+      # @param timeout [Numeric, nil]
+      #   Optional timeout, in seconds, used when waiting for worker results and
+      #   during shutdown. `nil` means wait indefinitely.
+      # @raise [UnsafeProcessorError]
+      #   Raised when the processor class has not declared `ractor_safe!`.
+      # @raise [ArgumentError]
+      #   Raised by {Configuration} when `size` or `timeout` is invalid.
       # @return [void]
       def initialize(processor:, size: Etc.nprocessors, timeout: nil)
         validate_processor!(processor)
 
         @processor = ::Ractor.make_shareable(processor)
         @configuration = Configuration.new(size:, timeout:)
-        @workers = Array.new(@configuration.size) do
+        booted_workers = Array.new(@configuration.size) do
           build_worker(@processor)
-        end.freeze
+        end
+
+        @workers = booted_workers.map(&:first).freeze
+        @worker_inboxes = booted_workers.map(&:last).freeze
 
         @next_worker = 0
+        @dispatch_mutex = Mutex.new
         @shutdown = false
       end
 
       # Process one work item synchronously.
       #
+      # This is a convenience wrapper around {#process_many}. The work still
+      # executes inside a worker Ractor; the call blocks until the corresponding
+      # `CDC::Core::ProcessorResult` is available or until the optional timeout
+      # is reached.
+      #
       # @param item [Object]
+      #   Shareable work item, usually a `CDC::Core::ChangeEvent`.
+      # @raise [ShutdownError]
+      #   Raised when work is submitted after {#shutdown} has started.
       # @return [CDC::Core::ProcessorResult]
+      #   Normalized processor result. Processor exceptions are captured as
+      #   failure results rather than escaping directly from the worker Ractor.
       def process(item)
         process_many([item]).fetch(0)
       end
 
       # Process many work items using the pre-warmed worker pool.
       #
-      # Results are returned in the same order as the supplied work items.
+      # Each item is made shareable before dispatch. Items are assigned to worker
+      # inboxes using round-robin selection. Responses are collected through a
+      # per-call reply port and returned in the same order as the input array.
       #
       # @param items [Array<Object>]
+      #   Work items to process. Empty arrays are valid and return an empty
+      #   frozen array.
+      # @raise [ShutdownError]
+      #   Raised when work is submitted after {#shutdown} has started.
       # @return [Array<CDC::Core::ProcessorResult>]
+      #   Frozen array of normalized results, ordered to match `items`.
       def process_many(items)
-        raise ShutdownError, "processor pool has been shut down" if @shutdown
-
         work_items = items.map { |item| ::Ractor.make_shareable(item) }
         reply_port = ::Ractor::Port.new
 
-        work_items.each_with_index do |item, index|
-          next_worker.send([index, item, reply_port])
-        end
+        dispatch(work_items, reply_port)
 
         collect_results(reply_port, work_items.length)
       ensure
         reply_port&.close
       end
 
-      # Shut down the pool.
+      # Shut down the pool and wait for worker Ractors to exit.
+      #
+      # Shutdown is idempotent. The first caller signals all worker inboxes with
+      # a stop message and waits for workers to join. Later calls return without
+      # doing anything.
       #
       # @return [void]
       def shutdown
-        return if @shutdown
+        @dispatch_mutex.synchronize do
+          return if @shutdown
 
-        @shutdown = true
+          @shutdown = true
+          signal_workers
+        end
 
-        @workers.each do |worker|
-          worker.send(nil)
+        wait_for_workers
+      end
+
+      private
+
+      def dispatch(work_items, reply_port)
+        @dispatch_mutex.synchronize do
+          raise ShutdownError, "processor pool has been shut down" if @shutdown
+
+          work_items.each_with_index do |item, index|
+            next_worker_inbox.send([index, item, reply_port])
+          end
+        end
+      end
+
+      def signal_workers
+        @worker_inboxes.each do |inbox|
+          inbox.send(nil)
         rescue Ractor::ClosedError
           # Already stopped.
         end
       end
 
-      private
+      def wait_for_workers
+        if @configuration.timeout
+          wait_for_workers_with_timeout
+        else
+          @workers.each(&:join)
+        end
+      end
+
+      def wait_for_workers_with_timeout
+        timeout = @configuration.timeout
+        return unless timeout
+
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+
+        @workers.each do |worker|
+          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          break unless remaining.positive?
+
+          ::Timeout.timeout(remaining, TimeoutError) { worker.join }
+        rescue TimeoutError
+          break
+        end
+      end
 
       def validate_processor!(processor)
         return if processor.class.respond_to?(:ractor_safe?) &&
@@ -80,38 +238,57 @@ module CDC
               "#{processor.class} must declare ractor_safe!"
       end
 
-      def build_worker(processor) # rubocop:disable Metrics/MethodLength
-        ::Ractor.new(processor) do |safe_processor|
-          loop do
-            message = ::Ractor.receive
-            break if message.nil?
+      def build_worker(processor)
+        boot_port = ::Ractor::Port.new
+        worker = start_worker(processor, boot_port)
 
-            index, item, reply_port = message
+        [worker, boot_port.receive]
+      ensure
+        boot_port.close
+      end
 
-            response = begin
-              CDC::Parallel::ResultCollector.worker_success(
-                safe_processor.process(item)
-              )
-            rescue StandardError => e
-              CDC::Parallel::ResultCollector.worker_failure(e)
-            end
+      def start_worker(processor, boot_port)
+        ::Ractor.new(processor, boot_port) do |safe_processor, ready_port|
+          inbox = ::Ractor::Port.new
+          ready_port << inbox
 
-            begin
-              reply_port << [index, response]
-            rescue Ractor::ClosedError
-              # The caller may have timed out and closed the reply port.
-            end
+          CDC::Parallel::ProcessorPool.send(:run_worker_loop, safe_processor, inbox)
+        end
+      end
+
+      def self.run_worker_loop(safe_processor, inbox)
+        loop do
+          message = inbox.receive
+          break if message.nil?
+
+          index, item, reply_port = message
+          response = worker_response(safe_processor, item)
+
+          begin
+            reply_port << [index, response]
+          rescue Ractor::ClosedError
+            # The caller may have timed out and closed the reply port.
           end
         end
       end
+      private_class_method :run_worker_loop
 
-      def next_worker
-        worker = @workers[@next_worker]
+      def self.worker_response(safe_processor, item)
+        CDC::Parallel::ResultCollector.worker_success(
+          safe_processor.process(item)
+        )
+      rescue StandardError => e
+        CDC::Parallel::ResultCollector.worker_failure(e)
+      end
+      private_class_method :worker_response
+
+      def next_worker_inbox
+        inbox = @worker_inboxes[@next_worker]
 
         @next_worker += 1
-        @next_worker = 0 if @next_worker >= @workers.length
+        @next_worker = 0 if @next_worker >= @worker_inboxes.length
 
-        worker
+        inbox
       end
 
       def collect_results(reply_port, count)
@@ -135,7 +312,10 @@ module CDC
       end
 
       def collect_results_with_timeout(reply_port, results)
-        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @configuration.timeout
+        timeout = @configuration.timeout
+        return results.freeze unless timeout
+
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
 
         results.length.times do
           remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)

data/lib/cdc/parallel/result_collector.rb

@@ -2,22 +2,58 @@
 
 module CDC
   module Parallel
-    # Normalizes values returned by workers.
+    # Converts raw worker responses into `CDC::Core::ProcessorResult` objects.
+    #
+    # Ractors cannot freely share arbitrary mutable Ruby objects. Worker
+    # responses must therefore be normalized into shareable payloads before they
+    # cross back to the caller. `ResultCollector` owns that small translation
+    # boundary.
+    #
+    # The worker side uses {worker_success} and {worker_failure}. The caller side
+    # uses {normalize} to convert those payloads into the public result contract.
+    #
+    # @example Normalizing a processor return value
+    #   response = CDC::Parallel::ResultCollector.worker_success(value)
+    #   result = CDC::Parallel::ResultCollector.normalize(response)
+    #
+    # @example Normalizing a worker exception
+    #   response = CDC::Parallel::ResultCollector.worker_failure(error)
+    #   result = CDC::Parallel::ResultCollector.normalize(response)
+    #   result.failure? #=> true
+    #
+    # @api public
     class ResultCollector
+      # Internal marker used to identify serialized worker failures.
+      #
+      # @return [Symbol]
       FAILURE_MARKER = :__cdc_parallel_failure__
 
       # Build a shareable success payload that can safely cross a Ractor boundary.
       #
+      # If the processor already returned a `CDC::Core::ProcessorResult`, that
+      # result is preserved. Any other shareable value will later be wrapped in a
+      # success result by {normalize}.
+      #
       # @param value [Object]
+      #   Processor return value.
+      # @raise [Ractor::Error]
+      #   Raised by Ruby when the value cannot be made shareable.
       # @return [Object]
+      #   Shareable success payload.
       def self.worker_success(value)
         ::Ractor.make_shareable(value)
       end
 
       # Build a shareable failure payload that can safely cross a Ractor boundary.
       #
+      # Exceptions themselves are not used as the cross-Ractor payload. Instead,
+      # the class name, message, and backtrace are serialized into a simple hash
+      # that can be reconstructed as a {ProcessorExecutionError} by {normalize}.
+      #
       # @param error [Exception]
+      #   Exception raised inside a worker Ractor.
       # @return [Hash]
+      #   Shareable serialized failure payload.
       def self.worker_failure(error)
         ::Ractor.make_shareable(
           {
@@ -29,9 +65,14 @@ module CDC
         )
       end
 
-      # Normalize a worker return value into a ProcessorResult.
+      # Normalize a worker return value into a `CDC::Core::ProcessorResult`.
+      #
+      # Failure payloads become failed processor results containing a
+      # {ProcessorExecutionError}. Existing processor results are returned
+      # unchanged. Other values are wrapped in a successful processor result.
       #
       # @param value [Object]
+      #   Raw worker response.
       # @return [CDC::Core::ProcessorResult]
       def self.normalize(value)
         if worker_failure?(value)

data/lib/cdc/parallel/router.rb

@@ -2,10 +2,32 @@
 
 module CDC
   module Parallel
-    # Routes supported CDC objects to the correct runtime pool.
+    # Routes normalized `cdc-core` work items to the matching parallel runtime
+    # primitive.
+    #
+    # `Router` is deliberately small. It does not inspect source-specific
+    # payloads, apply filters, decode database values, or decide scheduling
+    # policy. Its responsibility is only to look at the already-normalized
+    # `cdc-core` object shape and forward it to the pool that knows how to
+    # process that shape.
+    #
+    # @example Routing a single event
+    #   router.process(change_event)
+    #
+    # @example Routing a transaction envelope
+    #   router.process(transaction_envelope)
+    #
+    # @see CDC::Parallel::ProcessorPool
+    # @see CDC::Parallel::TransactionPool
+    # @api public
     class Router
+      # Create a router for event and transaction work items.
+      #
       # @param processor_pool [ProcessorPool]
+      #   Pool used for individual `CDC::Core::ChangeEvent` objects.
       # @param transaction_pool [TransactionPool]
+      #   Pool used for `CDC::Core::TransactionEnvelope` objects.
+      # @return [void]
       def initialize(processor_pool:, transaction_pool:)
         @processor_pool = processor_pool
         @transaction_pool = transaction_pool
@@ -14,6 +36,9 @@ module CDC
       # Process a supported CDC work item.
       #
       # @param item [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
+      #   Normalized CDC work item.
+      # @raise [UnsupportedWorkItemError]
+      #   Raised when the item is not a supported `cdc-core` work item shape.
       # @return [CDC::Core::ProcessorResult]
       def process(item)
         case item

data/lib/cdc/parallel/runtime.rb

@@ -2,11 +2,56 @@
 
 module CDC
   module Parallel
-    # High-level Ractor runtime facade for cdc-core processors.
+    # High-level Ractor runtime facade for `cdc-core` processors.
+    #
+    # `Runtime` is the primary public entry point for applications that want to
+    # execute normalized CDC work items with `cdc-parallel`. It wires together a
+    # {ProcessorPool}, a {TransactionPool}, and a {Router} so callers can submit
+    # either a single `CDC::Core::ChangeEvent` or a
+    # `CDC::Core::TransactionEnvelope` through one object.
+    #
+    # Use this class when you want the default cdc-parallel behavior:
+    #
+    # * validate that the processor declared `ractor_safe!`
+    # * boot a fixed set of worker Ractors
+    # * route events and transaction envelopes to the right pool
+    # * return `CDC::Core::ProcessorResult` objects
+    # * shut down all worker resources together
+    #
+    # @example Processing a change event
+    #   runtime = CDC::Parallel::Runtime.new(
+    #     processor: AnalyticsProcessor.new,
+    #     size: 4,
+    #     timeout: 5
+    #   )
+    #
+    #   result = runtime.process(change_event)
+    #   result.success? #=> true
+    #
+    #   runtime.shutdown
+    #
+    # @example Processing a transaction envelope
+    #   result = runtime.process_transaction(transaction)
+    #
+    # @note `Runtime` is an execution facade, not a source adapter. It expects
+    #   work that has already been normalized into `cdc-core` primitives.
+    # @see CDC::Parallel::ProcessorPool
+    # @see CDC::Parallel::TransactionPool
+    # @see CDC::Parallel::Router
+    # @api public
     class Runtime
+      # Create a runtime with event and transaction pools.
+      #
       # @param processor [CDC::Core::Processor]
+      #   Ractor-safe processor used for both event and transaction processing.
       # @param size [Integer]
-      # @param timeout [Float, nil]
+      #   Number of worker Ractors per internal pool.
+      # @param timeout [Numeric, nil]
+      #   Optional timeout in seconds for result collection and shutdown waits.
+      # @raise [UnsafeProcessorError]
+      #   Raised when the processor class has not declared `ractor_safe!`.
+      # @raise [ArgumentError]
+      #   Raised when size or timeout is invalid.
       # @return [void]
       def initialize(processor:, size: Etc.nprocessors, timeout: nil)
         @processor_pool = ProcessorPool.new(processor:, size:, timeout:)
@@ -15,9 +60,18 @@ module CDC
         @shutdown = false
       end
 
-      # Process a ChangeEvent or TransactionEnvelope.
+      # Process a supported normalized CDC work item.
+      #
+      # Supported items are `CDC::Core::ChangeEvent` and
+      # `CDC::Core::TransactionEnvelope`. Unsupported objects raise
+      # {UnsupportedWorkItemError} from the router.
       #
       # @param item [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
+      #   Normalized CDC work item.
+      # @raise [ShutdownError]
+      #   Raised when called after {#shutdown}.
+      # @raise [UnsupportedWorkItemError]
+      #   Raised for objects that are not supported CDC work item shapes.
       # @return [CDC::Core::ProcessorResult]
       def process(item)
         raise ShutdownError, "runtime has been shut down" if @shutdown
@@ -25,7 +79,11 @@ module CDC
         @router.process(item)
       end
 
-      # Alias for transaction-oriented processing.
+      # Process a transaction envelope.
+      #
+      # This method is a readability alias for transaction-oriented call sites.
+      # It delegates to {#process}, so it has the same validation, shutdown, and
+      # result behavior.
       #
       # @param transaction [CDC::Core::TransactionEnvelope]
       # @return [CDC::Core::ProcessorResult]
@@ -35,6 +93,9 @@ module CDC
 
       # Shut down all runtime resources.
       #
+      # Shutdown is idempotent and cascades to the internal event and transaction
+      # pools. After shutdown, {#process} raises {ShutdownError}.
+      #
       # @return [void]
       def shutdown
         return if @shutdown

data/lib/cdc/parallel/transaction_pool.rb

@@ -2,26 +2,81 @@
 
 module CDC
   module Parallel
-    # Processes a TransactionEnvelope as a single ordering-preserving unit.
+    # Processes a `CDC::Core::TransactionEnvelope` as one transaction-oriented
+    # work unit.
+    #
+    # `TransactionPool` uses {ProcessorPool} to process the events inside an
+    # envelope and then collapses the event-level results into one
+    # `CDC::Core::ProcessorResult` for the whole transaction.
+    #
+    # This class preserves the transaction boundary at the API level: callers
+    # submit a transaction envelope and receive a single success or failure
+    # result. Event results inside the transaction are still produced by the
+    # configured processor and are returned as the success value when every event
+    # succeeds.
+    #
+    # @example Processing a transaction envelope
+    #   pool = CDC::Parallel::TransactionPool.new(
+    #     processor: AuditProcessor.new,
+    #     size: 4
+    #   )
+    #
+    #   result = pool.process(transaction)
+    #   result.success? #=> true
+    #
+    # @note This class preserves the transaction as a result boundary. More
+    #   advanced ordering, checkpointing, retry, and atomic sink semantics belong
+    #   to higher-level runtime/sink contracts.
+    # @see CDC::Parallel::ProcessorPool
+    # @api public
     class TransactionPool
+      # Create a transaction pool.
+      #
       # @param processor [CDC::Core::Processor]
+      #   Ractor-safe processor used for each event inside the transaction.
       # @param size [Integer]
-      # @param timeout [Float, nil]
+      #   Number of worker Ractors in the underlying processor pool.
+      # @param timeout [Numeric, nil]
+      #   Optional timeout in seconds for result collection and shutdown waits.
+      # @raise [UnsafeProcessorError]
+      #   Raised when the processor class has not declared `ractor_safe!`.
+      # @return [void]
       def initialize(processor:, size: Etc.nprocessors, timeout: nil)
         @processor_pool = ProcessorPool.new(processor:, size:, timeout:)
       end
 
       # Process all events inside a transaction envelope.
       #
+      # The returned result is successful only when every event result succeeds.
+      # If any event fails, the transaction result is a failure using the first
+      # failure error and the complete event-result list as context.
+      #
       # @param transaction [CDC::Core::TransactionEnvelope]
+      #   Transaction envelope whose `events` will be processed.
       # @return [CDC::Core::ProcessorResult]
+      #   Success containing the ordered event results, or failure containing the
+      #   first event error.
       def process(transaction)
-        results = transaction.events.map { |event| @processor_pool.process(event) }.freeze
-        ResultCollector.normalize(results)
+        results = @processor_pool.process_many(transaction.events).freeze
+        failure = results.find(&:failure?)
+
+        if failure
+          error = failure.error || ProcessorExecutionError.new(
+            original_class: "CDC::Core::ProcessorResult",
+            original_message: "failed processor result did not include an error"
+          )
+
+          return CDC::Core::ProcessorResult.failure(error, event: results)
+        end
+
+        CDC::Core::ProcessorResult.success(results)
       end
 
       # Shut down worker resources.
       #
+      # Delegates to the underlying {ProcessorPool}. Shutdown is idempotent
+      # because the underlying pool is idempotent.
+      #
       # @return [void]
       def shutdown
         @processor_pool.shutdown

data/lib/cdc/parallel/version.rb

@@ -3,6 +3,11 @@
 module CDC
   module Parallel
     # Current cdc-parallel version.
-    VERSION = "0.2.1"
+    #
+    # This constant is used by RubyGems and by applications that need to inspect
+    # the loaded runtime version at boot time.
+    #
+    # @return [String]
+    VERSION = "0.2.3"
   end
 end