cdc-solid-queue 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c617d6e5257bc18cc38e242fe0ad0ac8b885de15fe6d6bdfbf5df5965924495
-  data.tar.gz: 2f8a7b90d7edd8f39010d74821730fc6ad833774bb54136c0dc51068f87ff9f0
+  metadata.gz: a31a7915a2b6ba29afe3497045f7be40477b0388b89a0357a18bb77428f5701c
+  data.tar.gz: 4485d2cdf5c8137dc7a11503364c3f05a1d00691d83d0be18f3c5a194ba6c968
 SHA512:
-  metadata.gz: f6c4cff53700e038fdfe3316c71972d61da38ca04937625143cc2790583e8cb64c7ee38d9f84da79f7425c04fdeef941c6dd65d41a78d0230cce7ba8c6ec55bb
-  data.tar.gz: '068730758e551e38b2f2abc65d9b79c21406dc243f00d022e3aea8885e96ddbb01cceb02c9b765c0ca5859e63f5ffe422435b1ee0bb3517840ac54873ae5a0ee'
+  metadata.gz: 9f3a59fef64e5227fcf6ddab9ce45363221058baa8dbed1932e7eedd75620fe2f1b4f0382c97232ad6cfc4f15ac30044f3427e05c21aebec2b4283aafb255f27
+  data.tar.gz: 6aacc4886cd318f2ce12dfcb6164d3c059c8a37064ef0daced35dc016d40a4047e2054e11b4f0a6d84a3af01447b060452f9e1c6e3dae5f1f187c53df1532035

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## Unreleased
+
+## 0.2.0
+
+- Optional downstream processor delegation to `cdc-concurrent` and `cdc-parallel`.
+- Rails example now demonstrates `cdc-concurrent` downstream processing.
+- Benchmark can measure direct downstream delegation overhead.
+
+## 0.1.2
+
+- Minimal Rails app example.
+- Local smoke tests.
+- Enqueue overhead benchmark.
+
 ## 0.1.1
 
 - Initial implementation skeleton.

data/README.md

@@ -54,6 +54,44 @@ class supports it. When `preserve_order` is enabled, the enqueued payload also
 includes cdc-solid-queue metadata with the configured ordering key and computed
 ordering value.
 
+## Downstream Processing
+
+Processor jobs can delegate work to CDC downstream runtime primitives. The
+default downstream runtime is `:concurrent`, backed by `cdc-concurrent`, which
+fits Solid Queue jobs that spend most of their time on I/O. CPU-heavy work can
+opt into `:parallel`, backed by `cdc-parallel`, in Ruby 4 applications.
+
+```ruby
+class WebhookProcessor < CDC::Core::Processor
+  concurrent_safe!
+
+  def process(event)
+    # perform I/O-bound work
+    CDC::Core::ProcessorResult.success(event)
+  end
+end
+
+CDC::SolidQueue.configure do |config|
+  config.processor_job = UserChangedJob
+  config.downstream_processor = WebhookProcessor.new
+  config.downstream_runtime = :concurrent
+  config.downstream_options = { concurrency: 100, timeout: 5.0 }
+end
+```
+
+Use `:parallel` only when the processor is Ractor-safe and the application runs
+on Ruby 4:
+
+```ruby
+config.downstream_runtime = :parallel
+config.downstream_options = { size: 4, timeout: 5 }
+```
+
+Both runtime gems are optional. Add `cdc-concurrent` or `cdc-parallel` to the
+application Gemfile when selecting that runtime. Without a configured
+`downstream_processor`, `CDC::SolidQueue::ProcessorJob` falls back to the job's
+own `#process(event)` method.
+
 ## Rails Task
 
 Rails applications can load the Railtie integration:
@@ -72,6 +110,71 @@ The task wires `Pgoutput::Client::Runner`, `Pgoutput::RelationTracker`,
 `Pgoutput::Decoder`, and `Pgoutput::SourceAdapter::Cdc` into the
 `CDC::SolidQueue::Runner`.
 
+See `examples/rails_app` for a minimal Rails-side setup with a Solid Queue job,
+initializer, Railtie require, and a local PostgreSQL container configured for
+logical replication.
+
+## Smoke Tests
+
+Run local smoke tests without PostgreSQL or Rails:
+
+```bash
+bundle exec rake smoke:local
+```
+
+The smoke tests verify enqueue metadata, event rehydration, and
+checkpoint-after-enqueue behavior.
+
+## Benchmark
+
+Run the enqueue overhead benchmark:
+
+```bash
+bundle exec rake benchmark:enqueue
+```
+
+Set `CDC_SOLID_QUEUE_BENCH_EVENTS` to control the event count.
+Set `CDC_SOLID_QUEUE_BENCH_MODE=downstream_direct` to measure direct downstream
+processor delegation overhead without Solid Queue enqueue translation.
+
+Example local result on Ruby 3.4.9:
+
+```text
+events=1000000 elapsed=15.7210s rate=63609.14 events/s
+```
+
+This is an upper-bound microbenchmark for the Ruby-side enqueue translation
+layer. It measures event serialization, queue and ordering metadata calculation,
+and dispatch to a fake benchmark job. It does not measure real Solid Queue
+database inserts, Rails job execution, PostgreSQL replication, pgoutput
+decoding, network I/O, or checkpoint persistence.
+
+In that run, `cdc-solid-queue` translated and dispatched about 63.6k synthetic
+events per second, so real throughput will usually be dominated by Solid Queue
+persistence, database latency, job execution cost, and CDC source throughput.
+
+Example `downstream_direct` results on the same machine:
+
+```text
+mode=downstream_direct events=100000000 elapsed=16.2669s rate=6147457.32 events/s
+mode=downstream_direct events=1000000000 elapsed=157.8708s rate=6334292.58 events/s
+```
+
+These runs measure the lowest-overhead downstream delegation path:
+
+```text
+CDC::SolidQueue::DownstreamProcessor
+  -> :direct runtime branch
+  -> BenchmarkProcessor#process(event)
+```
+
+They do not measure Solid Queue enqueueing, Active Job serialization,
+PostgreSQL CDC, pgoutput parsing or decoding, `cdc-concurrent`,
+`cdc-parallel`, real application processor work, network I/O, or database I/O.
+The result means the direct downstream adapter can dispatch about 6.1M to 6.3M
+prebuilt synthetic events per second on that machine, making the adapter layer
+negligible compared with real persistence, CDC source, and processor costs.
+
 ## MVP Checkpoint Rule
 
 A checkpoint advances after the Solid Queue job is durably inserted. Job execution success is handled by Solid Queue retry semantics.

data/lib/cdc/solid_queue/cli.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module CDC
+  module SolidQueue
+    # Command helpers used by Rails tasks and executable entrypoints.
+    module CLI
+      class << self
+        # Start PostgreSQL CDC ingestion using the global configuration.
+        #
+        # @return [Integer] number of enqueued events when the stream exits
+        def start
+          configuration = CDC::SolidQueue.configuration
+          enqueuer = CDC::SolidQueue::Enqueuer.new(configuration)
+          stream = CDC::SolidQueue::PostgresqlStream.new(configuration)
+
+          CDC::SolidQueue::Runner.new(stream:, enqueuer:).start
+        end
+      end
+    end
+  end
+end

data/lib/cdc/solid_queue/configuration.rb

@@ -12,8 +12,11 @@ module CDC
       SUPPORTED_SOURCE = :postgresql
       # Supported ordering scopes for serialized CDC events.
       ORDERING_KEYS = %i[identity primary_key relation transaction global none].freeze
+      # Supported downstream execution runtimes for processor jobs.
+      DOWNSTREAM_RUNTIMES = %i[concurrent parallel direct].freeze
 
-      attr_accessor :processor_job, :queue, :preserve_order, :ordering_key, :postgresql, :checkpoint
+      attr_accessor :processor_job, :queue, :preserve_order, :ordering_key, :postgresql, :checkpoint,
+                    :downstream_processor, :downstream_runtime, :downstream_options
 
       # Build a configuration with safe defaults.
       def initialize
@@ -23,6 +26,9 @@ module CDC
         @ordering_key = :identity
         @postgresql = {}
         @checkpoint = Checkpoint.new
+        @downstream_processor = nil
+        @downstream_runtime = :concurrent
+        @downstream_options = {}
       end
 
       # Validate this configuration.
@@ -37,6 +43,7 @@ module CDC
         validate_ordering_key!
         validate_postgresql!
         validate_checkpoint!
+        validate_downstream!
         true
       end
       # rubocop:enable Naming/PredicateMethod
@@ -81,6 +88,16 @@ module CDC
 
         raise ConfigurationError, 'checkpoint must respond to advance'
       end
+
+      def validate_downstream!
+        unless DOWNSTREAM_RUNTIMES.include?(@downstream_runtime)
+          raise ConfigurationError, "downstream_runtime must be one of: #{DOWNSTREAM_RUNTIMES.join(', ')}"
+        end
+
+        return if @downstream_processor.nil? || @downstream_processor.respond_to?(:process)
+
+        raise ConfigurationError, 'downstream_processor must respond to process'
+      end
     end
   end
 end

data/lib/cdc/solid_queue/downstream_processor.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module CDC
+  module SolidQueue
+    # Delegates processor-job work to CDC downstream runtime primitives.
+    class DownstreamProcessor
+      # @return [Configuration]
+      attr_reader :configuration
+
+      # @param configuration [Configuration]
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      # Process one normalized CDC work item.
+      #
+      # @param item [Object]
+      # @return [Object]
+      def process(item)
+        case configuration.downstream_runtime
+        when :direct
+          processor.process(item)
+        when :concurrent
+          process_with_runtime(concurrent_runtime, item)
+        when :parallel
+          process_with_runtime(parallel_runtime, item)
+        else
+          raise ConfigurationError, "unsupported downstream_runtime: #{configuration.downstream_runtime.inspect}"
+        end
+      end
+
+      private
+
+      def processor
+        configuration.downstream_processor || raise(ConfigurationError, 'downstream_processor is required')
+      end
+
+      def process_with_runtime(runtime, item)
+        runtime.process(item)
+      ensure
+        runtime.shutdown
+      end
+
+      def concurrent_runtime
+        require_runtime('cdc/concurrent', 'cdc-concurrent') unless defined?(CDC::Concurrent::Runtime)
+        CDC::Concurrent::Runtime.new(processor:, **configuration.downstream_options)
+      rescue LoadError => e
+        raise ConfigurationError, "cdc-concurrent is required for downstream_runtime :concurrent: #{e.message}"
+      end
+
+      def parallel_runtime
+        require_runtime('cdc/parallel', 'cdc-parallel') unless defined?(CDC::Parallel::Runtime)
+        CDC::Parallel::Runtime.new(processor:, **configuration.downstream_options)
+      rescue LoadError => e
+        raise ConfigurationError, "cdc-parallel is required for downstream_runtime :parallel: #{e.message}"
+      end
+
+      def require_runtime(feature, gem_name)
+        require feature
+      rescue LoadError
+        raise LoadError, "install #{gem_name} and require #{feature}"
+      end
+    end
+  end
+end

data/lib/cdc/solid_queue/processor_job.rb

@@ -20,7 +20,12 @@ module CDC
       # @param payload [Hash]
       # @return [Object] process return value
       def perform(payload)
-        process(EventSerializer.load_event(payload))
+        event = EventSerializer.load_event(payload)
+        if SolidQueue.configuration.downstream_processor
+          return DownstreamProcessor.new(SolidQueue.configuration).process(event)
+        end
+
+        process(event)
       end
 
       # Process a normalized CDC event payload.

data/lib/cdc/solid_queue/railtie.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
-require 'rails/railtie'
+require_relative '../solid_queue'
+begin
+  require 'rails/railtie'
+rescue LoadError
+  Rails::Railtie
+end
 
 module CDC
   module SolidQueue

data/lib/cdc/solid_queue/tasks/cdc_solid_queue.rake

@@ -3,10 +3,6 @@
 namespace :cdc_solid_queue do
   desc 'Start PostgreSQL CDC ingestion into Solid Queue'
   task start: :environment do
-    configuration = CDC::SolidQueue.configuration
-    enqueuer = CDC::SolidQueue::Enqueuer.new(configuration)
-    stream = CDC::SolidQueue::PostgresqlStream.new(configuration)
-
-    CDC::SolidQueue::Runner.new(stream:, enqueuer:).start
+    CDC::SolidQueue::CLI.start
   end
 end

data/lib/cdc/solid_queue/version.rb

@@ -3,6 +3,6 @@
 module CDC
   module SolidQueue
     # Current cdc-solid-queue gem version.
-    VERSION = '0.1.1'
+    VERSION = '0.2.0'
   end
 end

data/lib/cdc/solid_queue.rb

@@ -6,9 +6,11 @@ require_relative 'solid_queue/event_serializer'
 require_relative 'solid_queue/checkpoint'
 require_relative 'solid_queue/configuration'
 require_relative 'solid_queue/enqueuer'
+require_relative 'solid_queue/downstream_processor'
 require_relative 'solid_queue/processor_job'
 require_relative 'solid_queue/postgresql_stream'
 require_relative 'solid_queue/runner'
+require_relative 'solid_queue/cli'
 
 # Namespace for Change Data Capture integrations.
 module CDC

data/sig/cdc/solid_queue.rbs

@@ -32,6 +32,22 @@ module Pgoutput
 end
 
 module CDC
+  module Concurrent
+    class Runtime
+      def initialize: (processor: untyped, **untyped options) -> void
+      def process: (untyped item) -> untyped
+      def shutdown: () -> untyped
+    end
+  end
+
+  module Parallel
+    class Runtime
+      def initialize: (processor: untyped, **untyped options) -> void
+      def process: (untyped item) -> untyped
+      def shutdown: () -> untyped
+    end
+  end
+
   module SolidQueue
     VERSION: String
 
@@ -54,6 +70,7 @@ module CDC
     class Configuration
       SUPPORTED_SOURCE: Symbol
       ORDERING_KEYS: Array[Symbol]
+      DOWNSTREAM_RUNTIMES: Array[Symbol]
 
       attr_accessor processor_job: untyped
       attr_accessor queue: String
@@ -61,6 +78,9 @@ module CDC
       attr_accessor ordering_key: Symbol
       attr_accessor postgresql: Hash[Symbol, untyped]
       attr_accessor checkpoint: untyped
+      attr_accessor downstream_processor: untyped
+      attr_accessor downstream_runtime: Symbol
+      attr_accessor downstream_options: Hash[Symbol, untyped]
 
       def initialize: () -> void
       def validate!: () -> true
@@ -73,6 +93,7 @@ module CDC
       def validate_ordering_key!: () -> nil
       def validate_postgresql!: () -> nil
       def validate_checkpoint!: () -> nil
+      def validate_downstream!: () -> nil
     end
 
     class EventSerializer
@@ -116,6 +137,20 @@ module CDC
       def ordering_value: (Hash[untyped, untyped] payload) -> untyped
     end
 
+    class DownstreamProcessor
+      attr_reader configuration: Configuration
+      def initialize: (Configuration configuration) -> void
+      def process: (untyped item) -> untyped
+
+      private
+
+      def processor: () -> untyped
+      def process_with_runtime: (untyped runtime, untyped item) -> untyped
+      def concurrent_runtime: () -> untyped
+      def parallel_runtime: () -> untyped
+      def require_runtime: (String feature, String gem_name) -> untyped
+    end
+
     module ProcessorJob
       def self.included: (untyped base) -> void
       def perform: (Hash[untyped, untyped] payload) -> untyped
@@ -152,6 +187,10 @@ module CDC
       def metadata_value: (untyped metadata, Symbol name) -> untyped
     end
 
+    module CLI
+      def self.start: () -> Integer
+    end
+
     class Railtie < ::Rails::Railtie
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cdc-solid-queue
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -91,7 +91,9 @@ files:
 - README.md
 - lib/cdc/solid_queue.rb
 - lib/cdc/solid_queue/checkpoint.rb
+- lib/cdc/solid_queue/cli.rb
 - lib/cdc/solid_queue/configuration.rb
+- lib/cdc/solid_queue/downstream_processor.rb
 - lib/cdc/solid_queue/enqueuer.rb
 - lib/cdc/solid_queue/error.rb
 - lib/cdc/solid_queue/event_serializer.rb