cdc-solid-queue 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a31a7915a2b6ba29afe3497045f7be40477b0388b89a0357a18bb77428f5701c
-  data.tar.gz: 4485d2cdf5c8137dc7a11503364c3f05a1d00691d83d0be18f3c5a194ba6c968
+  metadata.gz: abd5d503ed994dcf6881b7636c24a2669d358e8e29ab3f0c88aef1e0236ed998
+  data.tar.gz: 36235e6ab5e44b9b40f75b5a6ca3cfc41fd03746dca852074ee14509c1ad4c07
 SHA512:
-  metadata.gz: 9f3a59fef64e5227fcf6ddab9ce45363221058baa8dbed1932e7eedd75620fe2f1b4f0382c97232ad6cfc4f15ac30044f3427e05c21aebec2b4283aafb255f27
-  data.tar.gz: 6aacc4886cd318f2ce12dfcb6164d3c059c8a37064ef0daced35dc016d40a4047e2054e11b4f0a6d84a3af01447b060452f9e1c6e3dae5f1f187c53df1532035
+  metadata.gz: e9d3c198e816c337815e2f7a840365b9176a789b9ab142f46ec9bc65f2414a9024c639f2e7d4b06ff2ce6e357d7e3b9376292997a435d9779f07f156ed8f7a3a
+  data.tar.gz: e7f0efe0daa73c5410eb52ea4b1076ec9c64b8022e80c5793bfccbd8b8540517c8ac8fe8d29b5cc0f6199400857a228661923e1918c0e3627035364ec347a106

data/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+## 0.3.0
+
+- Batch-sized enqueueing now feeds arrays into downstream `process_many`, so
+  the Solid Queue job path matches batch-style downstream runtimes.
+
 ## 0.2.0
 
 - Optional downstream processor delegation to `cdc-concurrent` and `cdc-parallel`.

data/README.md

@@ -52,7 +52,8 @@ end
 `config.queue` is applied through Active Job's `set(queue:)` API when the job
 class supports it. When `preserve_order` is enabled, the enqueued payload also
 includes cdc-solid-queue metadata with the configured ordering key and computed
-ordering value.
+ordering value. Set `config.batch_size` above `1` to enqueue multiple CDC
+events in one Solid Queue job and hand the batch to downstream `process_many`.
 
 ## Downstream Processing
 
@@ -90,7 +91,8 @@ config.downstream_options = { size: 4, timeout: 5 }
 Both runtime gems are optional. Add `cdc-concurrent` or `cdc-parallel` to the
 application Gemfile when selecting that runtime. Without a configured
 `downstream_processor`, `CDC::SolidQueue::ProcessorJob` falls back to the job's
-own `#process(event)` method.
+own `#process(event)` method, or `#process_many(events)` when a batched payload
+reaches a job that defines it.
 
 ## Rails Task
 
@@ -136,6 +138,9 @@ bundle exec rake benchmark:enqueue
 Set `CDC_SOLID_QUEUE_BENCH_EVENTS` to control the event count.
 Set `CDC_SOLID_QUEUE_BENCH_MODE=downstream_direct` to measure direct downstream
 processor delegation overhead without Solid Queue enqueue translation.
+Set `CDC_SOLID_QUEUE_BENCH_MODE=downstream_batch` to measure batched downstream
+delegation overhead. Set `CDC_SOLID_QUEUE_BENCH_BATCH_SIZE` to control the batch
+width.
 
 Example local result on Ruby 3.4.9:
 
@@ -175,6 +180,17 @@ The result means the direct downstream adapter can dispatch about 6.1M to 6.3M
 prebuilt synthetic events per second on that machine, making the adapter layer
 negligible compared with real persistence, CDC source, and processor costs.
 
+Batch mode example:
+
+```text
+mode=downstream_batch events=100000000 elapsed=... rate=... events/s
+```
+
+Batch mode measures one more layer: batch deserialization plus `process_many`
+dispatch through the downstream adapter. When a downstream runtime such as
+`cdc-concurrent` or `cdc-parallel` is configured, that batch is handed to the
+runtime pool in one call instead of event-by-event.
+
 ## MVP Checkpoint Rule
 
 A checkpoint advances after the Solid Queue job is durably inserted. Job execution success is handled by Solid Queue retry semantics.

data/lib/cdc/solid_queue/checkpoint.rb

@@ -27,6 +27,8 @@ module CDC
       private
 
       def position_for(event)
+        return event.map { |item| position_for(item) }.compact.last if event.is_a?(Array)
+
         payload = EventSerializer.dump(event)
         payload['source_position'] || payload['commit_lsn'] || payload.dig('metadata', 'wal_end_lsn')
       rescue SerializationError

data/lib/cdc/solid_queue/configuration.rb

@@ -16,7 +16,7 @@ module CDC
       DOWNSTREAM_RUNTIMES = %i[concurrent parallel direct].freeze
 
       attr_accessor :processor_job, :queue, :preserve_order, :ordering_key, :postgresql, :checkpoint,
-                    :downstream_processor, :downstream_runtime, :downstream_options
+                    :downstream_processor, :downstream_runtime, :downstream_options, :batch_size
 
       # Build a configuration with safe defaults.
       def initialize
@@ -29,6 +29,7 @@ module CDC
         @downstream_processor = nil
         @downstream_runtime = :concurrent
         @downstream_options = {}
+        @batch_size = 1
       end
 
       # Validate this configuration.
@@ -44,6 +45,7 @@ module CDC
         validate_postgresql!
         validate_checkpoint!
         validate_downstream!
+        validate_batch_size!
         true
       end
       # rubocop:enable Naming/PredicateMethod
@@ -98,6 +100,12 @@ module CDC
 
         raise ConfigurationError, 'downstream_processor must respond to process'
       end
+
+      def validate_batch_size!
+        return if @batch_size.is_a?(Integer) && @batch_size.positive?
+
+        raise ConfigurationError, 'batch_size must be a positive Integer'
+      end
     end
   end
 end

data/lib/cdc/solid_queue/downstream_processor.rb

@@ -14,16 +14,26 @@ module CDC
 
       # Process one normalized CDC work item.
       #
-      # @param item [Object]
+      # @param item [Object, Array<Object>]
       # @return [Object]
       def process(item)
+        return process_many(item) if item.is_a?(Array)
+
+        process_one(item)
+      end
+
+      # Process many normalized CDC work items.
+      #
+      # @param items [Array<Object>]
+      # @return [Object]
+      def process_many(items)
         case configuration.downstream_runtime
         when :direct
-          processor.process(item)
+          process_many_direct(items)
         when :concurrent
-          process_with_runtime(concurrent_runtime, item)
+          process_with_runtime(concurrent_runtime, items)
         when :parallel
-          process_with_runtime(parallel_runtime, item)
+          process_with_runtime(parallel_runtime, items)
         else
           raise ConfigurationError, "unsupported downstream_runtime: #{configuration.downstream_runtime.inspect}"
         end
@@ -35,12 +45,37 @@ module CDC
         configuration.downstream_processor || raise(ConfigurationError, 'downstream_processor is required')
       end
 
-      def process_with_runtime(runtime, item)
-        runtime.process(item)
+      def process_one(item)
+        case configuration.downstream_runtime
+        when :direct
+          processor.process(item)
+        when :concurrent
+          unwrap_single_result(process_with_runtime(concurrent_runtime, [item]))
+        when :parallel
+          unwrap_single_result(process_with_runtime(parallel_runtime, [item]))
+        else
+          raise ConfigurationError, "unsupported downstream_runtime: #{configuration.downstream_runtime.inspect}"
+        end
+      end
+
+      def process_many_direct(items)
+        return processor.process_many(items) if processor.respond_to?(:process_many)
+
+        items.map { |item| processor.process(item) }
+      end
+      private :process_many_direct
+
+      def process_with_runtime(runtime, items)
+        runtime.process_many(items)
       ensure
         runtime.shutdown
       end
 
+      def unwrap_single_result(result)
+        result.is_a?(Array) && result.length == 1 ? result.first : result
+      end
+      private :unwrap_single_result
+
       def concurrent_runtime
         require_runtime('cdc/concurrent', 'cdc-concurrent') unless defined?(CDC::Concurrent::Runtime)
         CDC::Concurrent::Runtime.new(processor:, **configuration.downstream_options)

data/lib/cdc/solid_queue/enqueuer.rb

@@ -15,10 +15,10 @@ module CDC
 
       # Enqueue one CDC event.
       #
-      # @param event [Object, Hash]
+      # @param event [Object, Hash, Array<Object>]
       # @return [Object] Active Job return value
       def enqueue(event)
-        payload = EventSerializer.dump(event)
+        payload = payload_for(event)
         payload = EventSerializer.with_enqueue_metadata(payload, enqueue_metadata(payload))
         job = configuration.processor_job
         return async_job(job).perform_later(payload) if job.respond_to?(:perform_later)
@@ -39,14 +39,25 @@ module CDC
           'queue' => configuration.queue,
           'preserve_order' => configuration.preserve_order,
           'ordering_key' => configuration.ordering_key,
-          'ordering_value' => ordering_value(payload)
+          'ordering_value' => ordering_value(payload),
+          'batch_size' => configuration.batch_size
         }
       end
 
       def ordering_value(payload)
         return nil unless configuration.preserve_order
 
-        EventSerializer.ordering_value(payload, configuration.ordering_key)
+        if payload.is_a?(Array)
+          payload.map { |event| EventSerializer.ordering_value(event, configuration.ordering_key) }
+        else
+          EventSerializer.ordering_value(payload, configuration.ordering_key)
+        end
+      end
+
+      def payload_for(event)
+        return EventSerializer.dump_batch(event) if event.is_a?(Array)
+
+        EventSerializer.dump(event)
       end
     end
   end

data/lib/cdc/solid_queue/event_serializer.rb

@@ -6,9 +6,18 @@ module CDC
     #
     # Payloads are plain hashes so Active Job can serialize them without needing
     # to load the original event object in the queue database.
+    # rubocop:disable Metrics/ClassLength
     class EventSerializer
       # Reserved payload key for cdc-solid-queue enqueue metadata.
       INTERNAL_METADATA_KEY = '_cdc_solid_queue'
+      # Lookup table for ordering value extraction by ordering key.
+      ORDERING_VALUE_FETCHERS = {
+        identity: ->(payload) { payload['identity'] || payload['primary_key'] },
+        primary_key: ->(payload) { payload['identity'] || payload['primary_key'] },
+        relation: ->(payload) { [payload['namespace'] || payload['schema'], payload['entity'] || payload['table']] },
+        transaction: ->(payload) { payload['transaction_id'] },
+        global: ->(payload) { payload['source_position'] || payload['commit_lsn'] }
+      }.freeze
 
       # Serialize an event-like object.
       #
@@ -27,6 +36,16 @@ module CDC
         normalize_hash(payload)
       end
 
+      # Serialize a batch of event-like objects.
+      #
+      # @param events [Array<Object>]
+      # @return [Array<Hash>]
+      def self.dump_batch(events)
+        raise SerializationError, 'events must be an Array' unless events.is_a?(Array)
+
+        events.map { |event| dump(event) }
+      end
+
       # Load a serialized event payload.
       #
       # @param payload [Hash]
@@ -38,11 +57,23 @@ module CDC
         strip_internal_metadata(normalize_hash(payload))
       end
 
+      # Load a batch of serialized event payloads.
+      #
+      # @param payloads [Array<Hash>]
+      # @return [Array<Hash>]
+      def self.load_batch(payloads)
+        raise SerializationError, 'payloads must be an Array' unless payloads.is_a?(Array)
+
+        payloads.map { |payload| load(payload) }
+      end
+
       # Load a serialized event payload into a CDC event when possible.
       #
       # @param payload [Hash]
       # @return [CDC::Core::ChangeEvent, Hash]
       def self.load_event(payload)
+        return load_batch(payload).map { |item| load_event(item) } if payload.is_a?(Array)
+
         normalized = load(payload)
         return normalized unless change_event_payload?(normalized)
 
@@ -55,6 +86,12 @@ module CDC
       # @param metadata [Hash]
       # @return [Hash]
       def self.with_enqueue_metadata(payload, metadata)
+        if payload.is_a?(Array)
+          return payload.each_with_index.map do |child, index|
+            with_enqueue_metadata(child, metadata_for_batch_item(metadata, index))
+          end
+        end
+
         normalized = normalize_hash(payload)
         normalized.merge(INTERNAL_METADATA_KEY => normalize_hash(metadata))
       end
@@ -64,6 +101,8 @@ module CDC
       # @param payload [Hash]
       # @return [Hash]
       def self.enqueue_metadata(payload)
+        return enqueue_metadata_for_batch(payload) if payload.is_a?(Array)
+
         normalized = normalize_hash(payload)
         metadata = normalized[INTERNAL_METADATA_KEY]
         metadata.is_a?(Hash) ? metadata : {}
@@ -75,19 +114,13 @@ module CDC
       # @param key [Symbol]
       # @return [Object, nil]
       def self.ordering_value(payload, key)
-        normalized = load(payload)
-        case key
-        when :identity, :primary_key
-          normalized['identity'] || normalized['primary_key']
-        when :relation
-          [normalized['namespace'] || normalized['schema'], normalized['entity'] || normalized['table']]
-        when :transaction
-          normalized['transaction_id']
-        when :global
-          normalized['source_position'] || normalized['commit_lsn']
-        when :none
-          nil
-        end
+        return payload.map { |item| ordering_value(item, key) } if payload.is_a?(Array)
+        return nil if key == :none
+
+        fetcher = ORDERING_VALUE_FETCHERS[key]
+        return nil unless fetcher
+
+        fetcher.call(load(payload))
       end
 
       # Normalize hash keys to strings recursively.
@@ -138,6 +171,25 @@ module CDC
         payload.key?('operation') && payload.key?('schema') && payload.key?('table')
       end
       private_class_method :change_event_payload?
+
+      def self.enqueue_metadata_for_batch(payloads)
+        payloads.each_with_index.map do |payload, index|
+          enqueue_metadata(payload).merge(
+            'batch_size' => payloads.length,
+            'batch_index' => index
+          )
+        end
+      end
+      private_class_method :enqueue_metadata_for_batch
+
+      def self.metadata_for_batch_item(metadata, index)
+        normalize_hash(metadata).merge(
+          'batch_size' => normalize_hash(metadata).fetch('batch_size'),
+          'batch_index' => index
+        )
+      end
+      private_class_method :metadata_for_batch_item
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/cdc/solid_queue/processor_job.rb

@@ -17,7 +17,7 @@ module CDC
 
       # Active Job entrypoint.
       #
-      # @param payload [Hash]
+      # @param payload [Hash, Array<Hash>]
       # @return [Object] process return value
       def perform(payload)
         event = EventSerializer.load_event(payload)
@@ -25,12 +25,14 @@ module CDC
           return DownstreamProcessor.new(SolidQueue.configuration).process(event)
         end
 
+        return process_many(event) if event.is_a?(Array) && respond_to?(:process_many)
+
         process(event)
       end
 
       # Process a normalized CDC event payload.
       #
-      # @param event [Hash]
+      # @param event [Hash, Array<Hash>]
       # @raise [NotImplementedError] when the including job does not override it
       def process(event)
         raise NotImplementedError, "#{self.class} must implement #process"

data/lib/cdc/solid_queue/runner.rb

@@ -21,17 +21,31 @@ module CDC
       #
       # @return [Integer] number of enqueued events
       def start
+        # @type var batch: Array[untyped]
+        batch = []
         count = 0
+
         @stream.each do |event|
-          result = @enqueuer.enqueue(event)
-          checkpoint(event, result)
-          count += 1
+          batch << event
+          next unless batch.length >= @enqueuer.configuration.batch_size
+
+          count += flush_batch(batch)
+          batch = []
         end
-        count
+
+        count + flush_batch(batch)
       end
 
       private
 
+      def flush_batch(batch)
+        return 0 if batch.empty?
+
+        result = @enqueuer.enqueue(batch.length == 1 ? batch.first : batch.dup)
+        checkpoint(batch, result)
+        batch.length
+      end
+
       def checkpoint(event, result)
         store = @enqueuer.configuration.checkpoint
         store&.advance(event, result)

data/lib/cdc/solid_queue/version.rb

@@ -3,6 +3,6 @@
 module CDC
   module SolidQueue
     # Current cdc-solid-queue gem version.
-    VERSION = '0.2.0'
+    VERSION = '0.3.0'
   end
 end

data/sig/cdc/solid_queue.rbs

@@ -81,6 +81,7 @@ module CDC
       attr_accessor downstream_processor: untyped
       attr_accessor downstream_runtime: Symbol
       attr_accessor downstream_options: Hash[Symbol, untyped]
+      attr_accessor batch_size: Integer
 
       def initialize: () -> void
       def validate!: () -> true
@@ -94,17 +95,21 @@ module CDC
       def validate_postgresql!: () -> nil
       def validate_checkpoint!: () -> nil
       def validate_downstream!: () -> nil
+      def validate_batch_size!: () -> nil
     end
 
     class EventSerializer
       INTERNAL_METADATA_KEY: String
+      ORDERING_VALUE_FETCHERS: Hash[Symbol, Proc]
 
       def self.dump: (untyped event) -> Hash[String, untyped]
+      def self.dump_batch: (::Array[untyped] events) -> ::Array[Hash[String, untyped]]
       def self.load: (Hash[untyped, untyped] payload) -> Hash[String, untyped]
-      def self.load_event: (Hash[untyped, untyped] payload) -> untyped
-      def self.with_enqueue_metadata: (Hash[untyped, untyped] payload, Hash[untyped, untyped] metadata) -> Hash[String, untyped]
-      def self.enqueue_metadata: (Hash[untyped, untyped] payload) -> Hash[String, untyped]
-      def self.ordering_value: (Hash[untyped, untyped] payload, Symbol key) -> untyped
+      def self.load_batch: (::Array[Hash[untyped, untyped]] payloads) -> ::Array[Hash[String, untyped]]
+      def self.load_event: (Hash[untyped, untyped] | ::Array[Hash[untyped, untyped]]) -> untyped
+      def self.with_enqueue_metadata: (Hash[untyped, untyped] | ::Array[Hash[untyped, untyped]], Hash[untyped, untyped]) -> untyped
+      def self.enqueue_metadata: (Hash[untyped, untyped] | ::Array[Hash[untyped, untyped]]) -> untyped
+      def self.ordering_value: (Hash[untyped, untyped] | ::Array[Hash[untyped, untyped]], Symbol) -> untyped
 
       private
 
@@ -112,6 +117,8 @@ module CDC
       def self.build_change_event: (Hash[String, untyped] normalized) -> untyped
       def self.strip_internal_metadata: (Hash[String, untyped] payload) -> Hash[String, untyped]
       def self.change_event_payload?: (Hash[String, untyped] payload) -> bool
+      def self.enqueue_metadata_for_batch: (::Array[Hash[String, untyped]] payloads) -> ::Array[Hash[String, untyped]]
+      def self.metadata_for_batch_item: (Hash[untyped, untyped] metadata, Integer index) -> Hash[String, untyped]
     end
 
     class Checkpoint
@@ -135,17 +142,22 @@ module CDC
       def async_job: (untyped job) -> untyped
       def enqueue_metadata: (Hash[untyped, untyped] payload) -> Hash[String, untyped]
       def ordering_value: (Hash[untyped, untyped] payload) -> untyped
+      def payload_for: (untyped event) -> untyped
     end
 
     class DownstreamProcessor
       attr_reader configuration: Configuration
       def initialize: (Configuration configuration) -> void
       def process: (untyped item) -> untyped
+      def process_many: (::Array[untyped] items) -> untyped
 
       private
 
       def processor: () -> untyped
-      def process_with_runtime: (untyped runtime, untyped item) -> untyped
+      def process_one: (untyped item) -> untyped
+      def process_many_direct: (::Array[untyped] items) -> untyped
+      def process_with_runtime: (untyped runtime, ::Array[untyped] items) -> untyped
+      def unwrap_single_result: (untyped result) -> untyped
       def concurrent_runtime: () -> untyped
       def parallel_runtime: () -> untyped
       def require_runtime: (String feature, String gem_name) -> untyped
@@ -154,7 +166,9 @@ module CDC
     module ProcessorJob
       def self.included: (untyped base) -> void
       def perform: (Hash[untyped, untyped] payload) -> untyped
+                 | (::Array[Hash[untyped, untyped]]) -> untyped
       def process: (Hash[String, untyped] event) -> untyped
+      def process_many: (::Array[untyped] events) -> untyped
     end
 
     class Runner
@@ -164,6 +178,7 @@ module CDC
       private
 
       def checkpoint: (untyped event, untyped result) -> untyped
+      def flush_batch: (::Array[untyped] batch) -> Integer
     end
 
     class PostgresqlStream

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cdc-solid-queue
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Ken C. Demanawa