cdc-concurrent 0.1.0 → 0.1.1

data/benchmark/processor_pool_benchmark.rb

@@ -0,0 +1,470 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "etc"
+require "json"
+require "socket"
+require "time"
+require "cdc_concurrent"
+require "cdc/core"
+
+# Reproducible processor-pool benchmark entrypoint.
+module CDCConcurrentBenchmark # rubocop:disable Metrics/ModuleLength
+  module_function
+
+  Config = Data.define(
+    :iterations,
+    :warmup,
+    :concurrency_counts,
+    :trials,
+    :min_duration,
+    :workload,
+    :batch_size,
+    :io_sleep
+  )
+  Trial = Data.define(:elapsed, :passes, :effective_events)
+
+  VALID_WORKLOADS = %w[tiny io batch].freeze
+
+  def integer_env(name, default)
+    value = ENV.fetch(name, default.to_s)
+    Integer(value)
+  rescue ArgumentError
+    warn "#{name} must be an integer; got #{value.inspect}"
+    exit 1
+  end
+
+  def positive_integer_env(name, default)
+    value = integer_env(name, default)
+    return value if value.positive?
+
+    warn "#{name} must be greater than zero; got #{value.inspect}"
+    exit 1
+  end
+
+  def nonnegative_integer_env(name, default)
+    value = integer_env(name, default)
+    return value if value >= 0
+
+    warn "#{name} must be zero or greater; got #{value.inspect}"
+    exit 1
+  end
+
+  def float_env(name, default)
+    value = ENV.fetch(name, default.to_s)
+    Float(value)
+  rescue ArgumentError
+    warn "#{name} must be numeric; got #{value.inspect}"
+    exit 1
+  end
+
+  def positive_float_env(name, default)
+    value = float_env(name, default)
+    return value if value.positive?
+
+    warn "#{name} must be greater than zero; got #{value.inspect}"
+    exit 1
+  end
+
+  def nonnegative_float_env(name, default)
+    value = float_env(name, default)
+    return value if value >= 0
+
+    warn "#{name} must be zero or greater; got #{value.inspect}"
+    exit 1
+  end
+
+  def concurrency_counts_env
+    value = ENV.fetch("BENCHMARK_CONCURRENCY_COUNTS", nil)
+    return [positive_integer_env("BENCHMARK_CONCURRENCY", 100)] unless value
+
+    counts = value.split(",").map { |entry| Integer(entry.strip) }
+    return counts if counts.any? && counts.all?(&:positive?)
+
+    warn "BENCHMARK_CONCURRENCY_COUNTS must contain positive integers; got #{value.inspect}"
+    exit 1
+  rescue ArgumentError
+    warn "BENCHMARK_CONCURRENCY_COUNTS must be a comma-separated integer list; got #{value.inspect}"
+    exit 1
+  end
+
+  def workload_env
+    workload = ENV.fetch("BENCHMARK_WORKLOAD", "io")
+
+    return workload if VALID_WORKLOADS.include?(workload)
+
+    warn "BENCHMARK_WORKLOAD must be one of #{VALID_WORKLOADS.join(", ")}; got #{workload.inspect}"
+    exit 1
+  end
+
+  def monotonic
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+
+  def config
+    Config.new(
+      iterations: positive_integer_env("BENCHMARK_ITERATIONS", 1_000),
+      warmup: nonnegative_integer_env("BENCHMARK_WARMUP", 100),
+      concurrency_counts: concurrency_counts_env,
+      trials: positive_integer_env("BENCHMARK_TRIALS", 5),
+      min_duration: positive_float_env("BENCHMARK_MIN_DURATION", 0.1),
+      workload: workload_env,
+      batch_size: positive_integer_env("BENCHMARK_BATCH_SIZE", 100),
+      io_sleep: nonnegative_float_env("BENCHMARK_IO_SLEEP", 0.001)
+    )
+  end
+
+  def change_event(counter)
+    CDC::Core::ChangeEvent.new(
+      operation: :update,
+      schema: "public",
+      table: "benchmark_events",
+      old_values: { "counter" => counter - 1 },
+      new_values: { "counter" => counter },
+      transaction_id: counter
+    )
+  end
+
+  def event
+    change_event(42)
+  end
+
+  def batch_event(settings)
+    Array.new(settings.batch_size) { |index| change_event(index + 1) }
+  end
+
+  # Minimal concurrent-safe processor.
+  #
+  # This intentionally benchmarks the overhead of the processor pool itself.
+  class TinyProcessor < CDC::Core::Processor
+    concurrent_safe!
+
+    def process(event)
+      payload = {
+        operation: event.operation,
+        schema: event.schema,
+        table: event.table,
+        changed: event.new_values.keys
+      }
+
+      CDC::Core::ProcessorResult.success(payload)
+    end
+  end
+
+  # I/O-like processor.
+  #
+  # The sleep call cooperates with Async's fiber scheduler inside the pool.
+  class IoProcessor < CDC::Core::Processor
+    concurrent_safe!
+
+    def initialize(sleep_seconds:)
+      @sleep_seconds = sleep_seconds
+      super()
+    end
+
+    def process(event)
+      sleep @sleep_seconds
+
+      CDC::Core::ProcessorResult.success(
+        {
+          operation: event.operation,
+          table: event.table,
+          waited_seconds: @sleep_seconds
+        }
+      )
+    end
+  end
+
+  # Batch I/O-like processor.
+  #
+  # This models one runtime dispatch that fans out over a group of CDC events.
+  class BatchIoProcessor < CDC::Core::Processor
+    concurrent_safe!
+
+    def initialize(sleep_seconds:)
+      @sleep_seconds = sleep_seconds
+      super()
+    end
+
+    def process(events)
+      sleep @sleep_seconds
+
+      CDC::Core::ProcessorResult.success(
+        {
+          count: events.length,
+          tables: events.map(&:table).uniq,
+          waited_seconds: @sleep_seconds
+        }
+      )
+    end
+  end
+
+  def processor_for(settings)
+    case settings.workload
+    when "tiny"
+      TinyProcessor.new
+    when "io"
+      IoProcessor.new(sleep_seconds: settings.io_sleep)
+    when "batch"
+      BatchIoProcessor.new(sleep_seconds: settings.io_sleep)
+    else
+      raise "unsupported workload: #{settings.workload}"
+    end
+  end
+
+  def sample_event_for(settings)
+    settings.workload == "batch" ? batch_event(settings) : event
+  end
+
+  def effective_events_per_pass(settings)
+    settings.workload == "batch" ? settings.iterations * settings.batch_size : settings.iterations
+  end
+
+  def run_trial(settings)
+    passes = 0
+    started_at = monotonic
+
+    loop do
+      yield
+      passes += 1
+      break if monotonic - started_at >= settings.min_duration
+    end
+
+    trial_for(settings, passes, monotonic - started_at)
+  end
+
+  def trial_for(settings, passes, elapsed)
+    Trial.new(
+      elapsed: elapsed,
+      passes: passes,
+      effective_events: passes * effective_events_per_pass(settings)
+    )
+  end
+
+  def run_trials(settings, &)
+    Array.new(settings.trials) { run_trial(settings, &) }
+  end
+
+  def serial_trials(settings, sample_event)
+    processor = processor_for(settings)
+    settings.warmup.times { processor.process(sample_event) }
+
+    run_trials(settings) do
+      settings.iterations.times do
+        result = processor.process(sample_event)
+        raise "serial processor failed" unless result.success?
+      end
+    end
+  end
+
+  def repeated_process_trials(settings, sample_event, concurrency)
+    with_pool(settings, concurrency) do |pool|
+      settings.warmup.times { pool.process(sample_event) }
+
+      run_trials(settings) do
+        settings.iterations.times do
+          result = pool.process(sample_event)
+          raise "pool.process failed" unless result.success?
+        end
+      end
+    end
+  end
+
+  def process_many_trials(settings, sample_event, concurrency)
+    with_pool(settings, concurrency) do |pool|
+      warmup_items = Array.new(settings.warmup) { sample_event }
+      benchmark_items = Array.new(settings.iterations) { sample_event }
+
+      pool.process_many(warmup_items)
+      run_trials(settings) do
+        results = pool.process_many(benchmark_items)
+        raise "pool.process_many failed" unless results.all?(&:success?)
+      end
+    end
+  end
+
+  def with_pool(settings, concurrency)
+    pool = CDC::Concurrent::ProcessorPool.new(
+      processor: processor_for(settings),
+      concurrency: concurrency
+    )
+
+    yield pool
+  ensure
+    pool&.shutdown
+  end
+
+  def report(settings, serial)
+    {
+      benchmark: "processor_pool",
+      gem: "cdc-concurrent",
+      timestamp: Time.now.utc.iso8601,
+      **report_body(settings, serial)
+    }
+  end
+
+  def report_body(settings, serial)
+    {
+      environment: environment,
+      config: config_report(settings),
+      workload_options: workload_options(settings),
+      serial: summarize_trials(serial),
+      concurrency_sweep: concurrency_sweep(settings, serial),
+      interpretation: interpretation
+    }
+  end
+
+  def config_report(settings)
+    {
+      iterations: settings.iterations,
+      warmup: settings.warmup,
+      trials: settings.trials,
+      min_duration_seconds: settings.min_duration,
+      concurrency_counts: settings.concurrency_counts,
+      workload: settings.workload
+    }
+  end
+
+  def environment
+    {
+      ruby: RUBY_DESCRIPTION,
+      ruby_engine: defined?(RUBY_ENGINE) ? RUBY_ENGINE : "ruby",
+      ruby_engine_version: defined?(RUBY_ENGINE_VERSION) ? RUBY_ENGINE_VERSION : RUBY_VERSION,
+      platform: RUBY_PLATFORM,
+      hostname: Socket.gethostname,
+      cpu_count: Etc.nprocessors,
+      uname: Etc.respond_to?(:uname) ? Etc.uname : {}
+    }
+  end
+
+  def workload_options(settings)
+    case settings.workload
+    when "tiny"
+      {}
+    when "io"
+      { io_sleep_seconds: settings.io_sleep }
+    when "batch"
+      { batch_size: settings.batch_size, io_sleep_seconds: settings.io_sleep }
+    end
+  end
+
+  def concurrency_sweep(settings, serial)
+    sample_event = sample_event_for(settings)
+
+    settings.concurrency_counts.map do |concurrency|
+      repeated = repeated_process_trials(settings, sample_event, concurrency)
+      many = process_many_trials(settings, sample_event, concurrency)
+
+      concurrency_report(concurrency, serial, repeated, many)
+    end
+  end
+
+  def concurrency_report(concurrency, serial, repeated, many)
+    {
+      concurrency: concurrency,
+      repeated_process: concurrent_summary(serial, repeated),
+      process_many: concurrent_summary(serial, many)
+    }
+  end
+
+  def concurrent_summary(serial, trials)
+    throughput_ratio = ratio(
+      serial_value: median(throughputs(serial)),
+      concurrent_value: median(throughputs(trials))
+    )
+    summary = summarize_trials(trials)
+
+    summary.merge(
+      ratio_to_serial_median_events_per_second: throughput_ratio,
+      interpretation: interpretation_for(throughput_ratio)
+    )
+  end
+
+  def summarize_trials(trials)
+    elapsed = trials.map(&:elapsed)
+    throughput = throughputs(trials)
+
+    {
+      trials: trials.length,
+      elapsed_seconds: distribution(elapsed),
+      events_per_second: distribution(throughput),
+      effective_events: distribution(trials.map(&:effective_events)),
+      passes: distribution(trials.map(&:passes)),
+      raw_trials: trials.map { |trial| trial_report(trial) }
+    }
+  end
+
+  def trial_report(trial)
+    {
+      elapsed_seconds: trial.elapsed.round(6),
+      effective_events: trial.effective_events,
+      passes: trial.passes,
+      events_per_second: (trial.effective_events / trial.elapsed).round(2)
+    }
+  end
+
+  def distribution(values)
+    sorted = values.sort
+
+    {
+      min: format_stat(sorted.first),
+      median: format_stat(median(sorted)),
+      max: format_stat(sorted.last),
+      p95: format_stat(percentile(sorted, 95))
+    }
+  end
+
+  def median(values)
+    sorted = values.sort
+    mid = sorted.length / 2
+
+    return sorted[mid] if sorted.length.odd?
+
+    (sorted[mid - 1] + sorted[mid]) / 2.0
+  end
+
+  def percentile(sorted_values, percentile)
+    index = ((percentile / 100.0) * (sorted_values.length - 1)).ceil
+    sorted_values[index]
+  end
+
+  def format_stat(value)
+    value.is_a?(Integer) ? value : value.round(6)
+  end
+
+  def throughputs(trials)
+    trials.map { |trial| trial.effective_events / trial.elapsed }
+  end
+
+  def ratio(serial_value:, concurrent_value:)
+    (concurrent_value / serial_value).round(4)
+  end
+
+  def interpretation_for(value)
+    if value > 1
+      "concurrent faster"
+    elsif value == 1
+      "concurrent equal to serial"
+    else
+      "serial faster"
+    end
+  end
+
+  def interpretation
+    {
+      "ratio > 1.0" => "concurrent median throughput is higher",
+      "ratio = 1.0" => "concurrent and serial median throughput are equivalent",
+      "ratio < 1.0" => "serial median throughput is higher"
+    }
+  end
+
+  def run
+    settings = config
+    sample_event = sample_event_for(settings)
+    serial = serial_trials(settings, sample_event)
+
+    puts JSON.pretty_generate(report(settings, serial))
+  end
+end
+
+CDCConcurrentBenchmark.run

data/lib/cdc/concurrent/configuration.rb

@@ -2,13 +2,24 @@
 
 module CDC
   module Concurrent
-    # Immutable configuration for concurrent runtimes.
+    # Immutable configuration for cdc-concurrent runtime pools.
+    #
+    # A Configuration instance captures the execution limits shared by
+    # ProcessorPool, TransactionPool, and Runtime. Instances are frozen so pool
+    # behavior cannot change while work is being processed.
     class Configuration
+      # @return [Integer] Maximum number of Async tasks allowed to run concurrently.
+      # @return [Float, nil] Optional per-event processing timeout in seconds.
+      # @return [Boolean] Whether batch results should be returned in input order.
       attr_reader :concurrency, :timeout, :preserve_order
 
-      # @param concurrency [Integer] maximum concurrent tasks.
-      # @param timeout [Float, nil] optional timeout.
-      # @param preserve_order [Boolean] whether batch results preserve input order.
+      # Builds a frozen runtime configuration.
+      #
+      # @param concurrency [Integer] Maximum number of Async tasks allowed to run at once.
+      # @param timeout [Float, nil] Optional per-event processing timeout in seconds.
+      # @param preserve_order [Boolean] Whether batch results should preserve input order.
+      # @raise [ArgumentError] If concurrency is not a positive Integer.
+      # @return [void] Does not return a useful value.
       def initialize(concurrency: 100, timeout: nil, preserve_order: true)
         raise ArgumentError, "concurrency must be an Integer" unless concurrency.is_a?(Integer)
         raise ArgumentError, "concurrency must be greater than zero" unless concurrency.positive?

data/lib/cdc/concurrent/processor_extensions.rb

@@ -4,21 +4,36 @@ module CDC
   # Optional concurrent runtime adapter for cdc-core processors.
   module Concurrent
     # Adds concurrent-safe declarations to CDC::Core::Processor subclasses.
+    #
+    # cdc-concurrent requires processors to explicitly opt in before they can be
+    # executed by Async-based runtime pools. The declaration is intentionally
+    # separate from ractor_safe! so processor authors can describe the execution
+    # model they support.
     module ProcessorExtensions
-      # Class methods added to CDC::Core::Processor.
+      # Class-level declarations added to CDC::Core::Processor.
       module ClassMethods
-        # Declare this processor safe for concurrent execution.
-        def concurrent_safe!
+        # Marks the processor class as safe for cdc-concurrent execution.
+        #
+        # This declaration means processor instances may be executed by the
+        # Async task fan-out runtime. It does not imply Ractor safety.
+        #
+        # @return [true] Always returns true after setting the declaration flag.
+        def concurrent_safe! # rubocop:disable Naming/PredicateMethod
           @concurrent_safe = true
+          true
         end
 
-        # @return [Boolean] whether instances are concurrent-safe.
+        # Reports whether the processor class declared concurrent_safe!.
+        #
+        # @return [Boolean] True when this processor class opted into concurrent execution.
         def concurrent_safe?
           @concurrent_safe == true
         end
       end
 
-      # @return [Boolean] whether this processor instance is concurrent-safe.
+      # Reports whether this processor instance is safe for concurrent execution.
+      #
+      # @return [Boolean] True when the processor class declared concurrent_safe!.
       def concurrent_safe?
         self.class.instance_variable_get(:@concurrent_safe) == true
       end
@@ -26,7 +41,7 @@ module CDC
 
     # Installs concurrent-safe declarations on CDC::Core::Processor.
     #
-    # @return [void]
+    # @return [void] Does not return a useful value.
     def self.install_processor_extensions!
       CDC::Core::Processor.extend(ProcessorExtensions::ClassMethods)
       CDC::Core::Processor.include(ProcessorExtensions)

data/lib/cdc/concurrent/processor_pool.rb

@@ -3,11 +3,62 @@
 module CDC
   module Concurrent
     # Executes one concurrent-safe processor using Async tasks.
+    #
+    # ARCHITECTURAL NOTE
+    #
+    # cdc-concurrent implements the same fan-out / fan-in execution pattern used
+    # by cdc-parallel. The runtime differs, but the processor contract and result
+    # contract remain the same.
+    #
+    #   events
+    #      |
+    #      v
+    #   fan-out
+    #      |
+    #      +----> Async task
+    #      +----> Async task
+    #      +----> Async task
+    #      |
+    #      v
+    #   fan-in
+    #      |
+    #      v
+    #   ProcessorResult array
+    #
+    # Fan-out:
+    #
+    # * Events are dispatched into Async tasks.
+    # * Async::Semaphore bounds the number of concurrently running tasks.
+    # * Multiple events may make progress concurrently under Ruby's scheduler.
+    #
+    # Fan-in:
+    #
+    # * Tasks append indexed ProcessorResult values into a shared collection.
+    # * Results may complete out of execution order.
+    # * When preserve_order is enabled, ProcessorPool sorts by submission index so
+    #   the returned array matches the input order.
+    #
+    # Relationship to cdc-parallel:
+    #
+    # * cdc-concurrent performs fan-out using Async tasks and cooperative
+    #   concurrency.
+    # * cdc-parallel performs fan-out using pre-warmed Ractor workers and true
+    #   parallel execution.
+    # * Both runtimes preserve the same processor contract and return
+    #   CDC::Core::ProcessorResult objects.
+    #
+    # Processor authors should be able to switch runtimes without changing
+    # processor behavior when their processor satisfies the selected runtime's
+    # safety declaration.
     class ProcessorPool
-      # @param processor [CDC::Core::Processor]
-      # @param concurrency [Integer]
-      # @param timeout [Float, nil]
-      # @param preserve_order [Boolean]
+      # Builds an Async-backed processor pool.
+      #
+      # @param processor [CDC::Core::Processor] Processor instance that declares concurrent_safe!.
+      # @param concurrency [Integer] Maximum number of Async tasks allowed to run at once.
+      # @param timeout [Float, nil] Optional per-event processing timeout in seconds.
+      # @param preserve_order [Boolean] Whether process_many should return results in input order.
+      # @raise [UnsafeProcessorError] If the processor does not declare concurrent_safe!.
+      # @return [void] Does not return a useful value.
       def initialize(processor:, concurrency: 100, timeout: nil, preserve_order: true)
         validate_processor!(processor)
 
@@ -16,16 +67,25 @@ module CDC
         @shutdown = false
       end
 
-      # @param event [CDC::Core::ChangeEvent]
-      # @return [CDC::Core::ProcessorResult]
+      # Processes one event synchronously through the Async runtime.
+      #
+      # @param event [CDC::Core::ChangeEvent] Event to process.
+      # @raise [ShutdownError] If the pool has already been shut down.
+      # @return [CDC::Core::ProcessorResult] Normalized processor result.
       def process(event)
         raise ShutdownError, "processor pool has been shut down" if @shutdown
 
         process_one(event)
       end
 
-      # @param events [Array<CDC::Core::ChangeEvent>]
-      # @return [Array<CDC::Core::ProcessorResult>]
+      # Processes many events through bounded Async fan-out.
+      #
+      # When preserve_order is true, the returned array matches the order of the
+      # supplied events even if individual tasks complete out of order.
+      #
+      # @param events [Array<CDC::Core::ChangeEvent>] Events to process.
+      # @raise [ShutdownError] If the pool has already been shut down.
+      # @return [Array<CDC::Core::ProcessorResult>] Frozen array of normalized results.
       def process_many(events)
         raise ShutdownError, "processor pool has been shut down" if @shutdown
         return empty_results if events.empty?
@@ -39,7 +99,9 @@ module CDC
         indexed_results.map(&:last).freeze
       end
 
-      # @return [void]
+      # Prevents new work from being submitted to the pool.
+      #
+      # @return [void] Does not return a useful value.
       def shutdown
         @shutdown = true
       end

data/lib/cdc/concurrent/result_collector.rb

@@ -2,10 +2,17 @@
 
 module CDC
   module Concurrent
-    # Normalizes values returned by concurrent workers.
+    # Normalizes values returned by Async tasks into ProcessorResult objects.
+    #
+    # ResultCollector keeps runtime pools tolerant of processors that either
+    # return a CDC::Core::ProcessorResult directly or return a plain Ruby value.
+    # Plain values are wrapped as successful ProcessorResult instances, while
+    # raised errors are represented as failure results.
     class ResultCollector
-      # @param value [Object]
-      # @return [CDC::Core::ProcessorResult]
+      # Normalizes a processor return value into a ProcessorResult.
+      #
+      # @param value [Object] Value returned by a processor or an existing ProcessorResult.
+      # @return [CDC::Core::ProcessorResult] Existing ProcessorResult or success result wrapping the value.
       def self.normalize(value)
         return value if value.is_a?(CDC::Core::ProcessorResult)
 
@@ -14,8 +21,10 @@ module CDC
         failure(e)
       end
 
-      # @param error [Exception]
-      # @return [CDC::Core::ProcessorResult]
+      # Wraps an exception as a failed ProcessorResult.
+      #
+      # @param error [Exception] Error raised while processing an event.
+      # @return [CDC::Core::ProcessorResult] Failure result containing the supplied error.
       def self.failure(error)
         CDC::Core::ProcessorResult.failure(error)
       end