cdc-solid-queue 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2a93bea7c30ca2f1c54bb53b00057967b662b16c8ca3a40bb5afb6c73ccb736
-  data.tar.gz: 1d413c1aeae90544b9430864d7cb753e1b510115a571d4640bf5c6a7c09caf20
+  metadata.gz: a31a7915a2b6ba29afe3497045f7be40477b0388b89a0357a18bb77428f5701c
+  data.tar.gz: 4485d2cdf5c8137dc7a11503364c3f05a1d00691d83d0be18f3c5a194ba6c968
 SHA512:
-  metadata.gz: 50b5a0edeec10b23ef6eeb124bb39e77d31a0148aa25128264936678c4dbd7b995f1d3e1cae3341bd1458c5ec687d1a579f291c0d714ebdb3c9de2ff5b790c08
-  data.tar.gz: c73e0438419bc58f93f0fe641df7fbefc266f39c60d322ceea0bad96e1524afa0197ce89d12e857bab2898f55513d29f98f3f0228fc526037046e0ac739e1a61
+  metadata.gz: 9f3a59fef64e5227fcf6ddab9ce45363221058baa8dbed1932e7eedd75620fe2f1b4f0382c97232ad6cfc4f15ac30044f3427e05c21aebec2b4283aafb255f27
+  data.tar.gz: 6aacc4886cd318f2ce12dfcb6164d3c059c8a37064ef0daced35dc016d40a4047e2054e11b4f0a6d84a3af01447b060452f9e1c6e3dae5f1f187c53df1532035

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+## 0.2.0
+
+- Optional downstream processor delegation to `cdc-concurrent` and `cdc-parallel`.
+- Rails example now demonstrates `cdc-concurrent` downstream processing.
+- Benchmark can measure direct downstream delegation overhead.
+
 ## 0.1.2
 
 - Minimal Rails app example.

data/README.md

@@ -54,6 +54,44 @@ class supports it. When `preserve_order` is enabled, the enqueued payload also
 includes cdc-solid-queue metadata with the configured ordering key and computed
 ordering value.
 
+## Downstream Processing
+
+Processor jobs can delegate work to CDC downstream runtime primitives. The
+default downstream runtime is `:concurrent`, backed by `cdc-concurrent`, which
+fits Solid Queue jobs that spend most of their time on I/O. CPU-heavy work can
+opt into `:parallel`, backed by `cdc-parallel`, in Ruby 4 applications.
+
+```ruby
+class WebhookProcessor < CDC::Core::Processor
+  concurrent_safe!
+
+  def process(event)
+    # perform I/O-bound work
+    CDC::Core::ProcessorResult.success(event)
+  end
+end
+
+CDC::SolidQueue.configure do |config|
+  config.processor_job = UserChangedJob
+  config.downstream_processor = WebhookProcessor.new
+  config.downstream_runtime = :concurrent
+  config.downstream_options = { concurrency: 100, timeout: 5.0 }
+end
+```
+
+Use `:parallel` only when the processor is Ractor-safe and the application runs
+on Ruby 4:
+
+```ruby
+config.downstream_runtime = :parallel
+config.downstream_options = { size: 4, timeout: 5 }
+```
+
+Both runtime gems are optional. Add `cdc-concurrent` or `cdc-parallel` to the
+application Gemfile when selecting that runtime. Without a configured
+`downstream_processor`, `CDC::SolidQueue::ProcessorJob` falls back to the job's
+own `#process(event)` method.
+
 ## Rails Task
 
 Rails applications can load the Railtie integration:
@@ -96,6 +134,8 @@ bundle exec rake benchmark:enqueue
 ```
 
 Set `CDC_SOLID_QUEUE_BENCH_EVENTS` to control the event count.
+Set `CDC_SOLID_QUEUE_BENCH_MODE=downstream_direct` to measure direct downstream
+processor delegation overhead without Solid Queue enqueue translation.
 
 Example local result on Ruby 3.4.9:
 
@@ -113,6 +153,28 @@ In that run, `cdc-solid-queue` translated and dispatched about 63.6k synthetic
 events per second, so real throughput will usually be dominated by Solid Queue
 persistence, database latency, job execution cost, and CDC source throughput.
 
+Example `downstream_direct` results on the same machine:
+
+```text
+mode=downstream_direct events=100000000 elapsed=16.2669s rate=6147457.32 events/s
+mode=downstream_direct events=1000000000 elapsed=157.8708s rate=6334292.58 events/s
+```
+
+These runs measure the lowest-overhead downstream delegation path:
+
+```text
+CDC::SolidQueue::DownstreamProcessor
+  -> :direct runtime branch
+  -> BenchmarkProcessor#process(event)
+```
+
+They do not measure Solid Queue enqueueing, Active Job serialization,
+PostgreSQL CDC, pgoutput parsing or decoding, `cdc-concurrent`,
+`cdc-parallel`, real application processor work, network I/O, or database I/O.
+The result means the direct downstream adapter can dispatch about 6.1M to 6.3M
+prebuilt synthetic events per second on that machine, making the adapter layer
+negligible compared with real persistence, CDC source, and processor costs.
+
 ## MVP Checkpoint Rule
 
 A checkpoint advances after the Solid Queue job is durably inserted. Job execution success is handled by Solid Queue retry semantics.

data/lib/cdc/solid_queue/configuration.rb

@@ -12,8 +12,11 @@ module CDC
       SUPPORTED_SOURCE = :postgresql
       # Supported ordering scopes for serialized CDC events.
       ORDERING_KEYS = %i[identity primary_key relation transaction global none].freeze
+      # Supported downstream execution runtimes for processor jobs.
+      DOWNSTREAM_RUNTIMES = %i[concurrent parallel direct].freeze
 
-      attr_accessor :processor_job, :queue, :preserve_order, :ordering_key, :postgresql, :checkpoint
+      attr_accessor :processor_job, :queue, :preserve_order, :ordering_key, :postgresql, :checkpoint,
+                    :downstream_processor, :downstream_runtime, :downstream_options
 
       # Build a configuration with safe defaults.
       def initialize
@@ -23,6 +26,9 @@ module CDC
         @ordering_key = :identity
         @postgresql = {}
         @checkpoint = Checkpoint.new
+        @downstream_processor = nil
+        @downstream_runtime = :concurrent
+        @downstream_options = {}
       end
 
       # Validate this configuration.
@@ -37,6 +43,7 @@ module CDC
         validate_ordering_key!
         validate_postgresql!
         validate_checkpoint!
+        validate_downstream!
         true
       end
       # rubocop:enable Naming/PredicateMethod
@@ -81,6 +88,16 @@ module CDC
 
         raise ConfigurationError, 'checkpoint must respond to advance'
       end
+
+      def validate_downstream!
+        unless DOWNSTREAM_RUNTIMES.include?(@downstream_runtime)
+          raise ConfigurationError, "downstream_runtime must be one of: #{DOWNSTREAM_RUNTIMES.join(', ')}"
+        end
+
+        return if @downstream_processor.nil? || @downstream_processor.respond_to?(:process)
+
+        raise ConfigurationError, 'downstream_processor must respond to process'
+      end
     end
   end
 end

data/lib/cdc/solid_queue/downstream_processor.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module CDC
+  module SolidQueue
+    # Delegates processor-job work to CDC downstream runtime primitives.
+    class DownstreamProcessor
+      # @return [Configuration]
+      attr_reader :configuration
+
+      # @param configuration [Configuration]
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      # Process one normalized CDC work item.
+      #
+      # @param item [Object]
+      # @return [Object]
+      def process(item)
+        case configuration.downstream_runtime
+        when :direct
+          processor.process(item)
+        when :concurrent
+          process_with_runtime(concurrent_runtime, item)
+        when :parallel
+          process_with_runtime(parallel_runtime, item)
+        else
+          raise ConfigurationError, "unsupported downstream_runtime: #{configuration.downstream_runtime.inspect}"
+        end
+      end
+
+      private
+
+      def processor
+        configuration.downstream_processor || raise(ConfigurationError, 'downstream_processor is required')
+      end
+
+      def process_with_runtime(runtime, item)
+        runtime.process(item)
+      ensure
+        runtime.shutdown
+      end
+
+      def concurrent_runtime
+        require_runtime('cdc/concurrent', 'cdc-concurrent') unless defined?(CDC::Concurrent::Runtime)
+        CDC::Concurrent::Runtime.new(processor:, **configuration.downstream_options)
+      rescue LoadError => e
+        raise ConfigurationError, "cdc-concurrent is required for downstream_runtime :concurrent: #{e.message}"
+      end
+
+      def parallel_runtime
+        require_runtime('cdc/parallel', 'cdc-parallel') unless defined?(CDC::Parallel::Runtime)
+        CDC::Parallel::Runtime.new(processor:, **configuration.downstream_options)
+      rescue LoadError => e
+        raise ConfigurationError, "cdc-parallel is required for downstream_runtime :parallel: #{e.message}"
+      end
+
+      def require_runtime(feature, gem_name)
+        require feature
+      rescue LoadError
+        raise LoadError, "install #{gem_name} and require #{feature}"
+      end
+    end
+  end
+end

data/lib/cdc/solid_queue/processor_job.rb

@@ -20,7 +20,12 @@ module CDC
       # @param payload [Hash]
       # @return [Object] process return value
       def perform(payload)
-        process(EventSerializer.load_event(payload))
+        event = EventSerializer.load_event(payload)
+        if SolidQueue.configuration.downstream_processor
+          return DownstreamProcessor.new(SolidQueue.configuration).process(event)
+        end
+
+        process(event)
       end
 
       # Process a normalized CDC event payload.

data/lib/cdc/solid_queue/version.rb

@@ -3,6 +3,6 @@
 module CDC
   module SolidQueue
     # Current cdc-solid-queue gem version.
-    VERSION = '0.1.2'
+    VERSION = '0.2.0'
   end
 end

data/lib/cdc/solid_queue.rb

@@ -6,6 +6,7 @@ require_relative 'solid_queue/event_serializer'
 require_relative 'solid_queue/checkpoint'
 require_relative 'solid_queue/configuration'
 require_relative 'solid_queue/enqueuer'
+require_relative 'solid_queue/downstream_processor'
 require_relative 'solid_queue/processor_job'
 require_relative 'solid_queue/postgresql_stream'
 require_relative 'solid_queue/runner'

data/sig/cdc/solid_queue.rbs

@@ -32,6 +32,22 @@ module Pgoutput
 end
 
 module CDC
+  module Concurrent
+    class Runtime
+      def initialize: (processor: untyped, **untyped options) -> void
+      def process: (untyped item) -> untyped
+      def shutdown: () -> untyped
+    end
+  end
+
+  module Parallel
+    class Runtime
+      def initialize: (processor: untyped, **untyped options) -> void
+      def process: (untyped item) -> untyped
+      def shutdown: () -> untyped
+    end
+  end
+
   module SolidQueue
     VERSION: String
 
@@ -54,6 +70,7 @@ module CDC
     class Configuration
       SUPPORTED_SOURCE: Symbol
       ORDERING_KEYS: Array[Symbol]
+      DOWNSTREAM_RUNTIMES: Array[Symbol]
 
       attr_accessor processor_job: untyped
       attr_accessor queue: String
@@ -61,6 +78,9 @@ module CDC
       attr_accessor ordering_key: Symbol
       attr_accessor postgresql: Hash[Symbol, untyped]
       attr_accessor checkpoint: untyped
+      attr_accessor downstream_processor: untyped
+      attr_accessor downstream_runtime: Symbol
+      attr_accessor downstream_options: Hash[Symbol, untyped]
 
       def initialize: () -> void
       def validate!: () -> true
@@ -73,6 +93,7 @@ module CDC
       def validate_ordering_key!: () -> nil
       def validate_postgresql!: () -> nil
       def validate_checkpoint!: () -> nil
+      def validate_downstream!: () -> nil
     end
 
     class EventSerializer
@@ -116,6 +137,20 @@ module CDC
       def ordering_value: (Hash[untyped, untyped] payload) -> untyped
     end
 
+    class DownstreamProcessor
+      attr_reader configuration: Configuration
+      def initialize: (Configuration configuration) -> void
+      def process: (untyped item) -> untyped
+
+      private
+
+      def processor: () -> untyped
+      def process_with_runtime: (untyped runtime, untyped item) -> untyped
+      def concurrent_runtime: () -> untyped
+      def parallel_runtime: () -> untyped
+      def require_runtime: (String feature, String gem_name) -> untyped
+    end
+
     module ProcessorJob
       def self.included: (untyped base) -> void
       def perform: (Hash[untyped, untyped] payload) -> untyped

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cdc-solid-queue
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -93,6 +93,7 @@ files:
 - lib/cdc/solid_queue/checkpoint.rb
 - lib/cdc/solid_queue/cli.rb
 - lib/cdc/solid_queue/configuration.rb
+- lib/cdc/solid_queue/downstream_processor.rb
 - lib/cdc/solid_queue/enqueuer.rb
 - lib/cdc/solid_queue/error.rb
 - lib/cdc/solid_queue/event_serializer.rb