mammoth 0.7.2 → 0.9.0

data/lib/mammoth/persisted_payload_deserializer.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Mammoth
+  # Reconstructs exact CDC core work items from persisted webhook payloads.
+  #
+  # Dead letters and sample files contain Mammoth's JSON delivery projection,
+  # not live CDC objects. This class is the explicit boundary that converts
+  # those persisted representations back into core vocabulary.
+  class PersistedPayloadDeserializer
+    class << self
+      # Deserialize one persisted event payload.
+      #
+      # @param payload [Hash] Mammoth event payload
+      # @return [CDC::Core::ChangeEvent] reconstructed core event
+      def event(payload)
+        attributes = stringify_keys(payload)
+        data = attributes["data"]
+        metadata = enriched_metadata(attributes)
+
+        CDC::Core::ChangeEvent.new(
+          operation: attributes.fetch("operation"),
+          schema: attributes["namespace"] || attributes.fetch("schema"),
+          table: attributes["entity"] || attributes.fetch("table"),
+          old_values: attributes["old_values"] || delete_values(attributes, data),
+          new_values: attributes["new_values"] || changed_values(attributes, data),
+          primary_key: attributes["identity"] || attributes["primary_key"],
+          transaction_id: attributes["transaction_id"],
+          commit_lsn: attributes["commit_lsn"] || attributes["source_position"],
+          sequence_number: attributes["sequence_number"],
+          occurred_at: parse_time(attributes["occurred_at"]),
+          metadata: metadata
+        )
+      rescue KeyError, ArgumentError, TypeError => e
+        raise ConfigurationError, "invalid persisted CDC event: #{e.message}"
+      end
+
+      # Deserialize one persisted transaction payload.
+      #
+      # @param payload [Hash] Mammoth transaction webhook payload
+      # @return [CDC::Core::TransactionEnvelope] reconstructed core transaction
+      def transaction(payload)
+        attributes = stringify_keys(payload)
+        CDC::Core::TransactionEnvelope.new(
+          transaction_id: attributes.fetch("transaction_id"),
+          events: attributes.fetch("events").map { |event_payload| event(event_payload) },
+          commit_lsn: attributes["commit_lsn"] || attributes["source_position"],
+          committed_at: parse_time(attributes["committed_at"]),
+          metadata: enriched_metadata(attributes)
+        )
+      rescue KeyError, ArgumentError, TypeError => e
+        raise ConfigurationError, "invalid persisted CDC transaction: #{e.message}"
+      end
+
+      private
+
+      def stringify_keys(payload)
+        payload.to_h.transform_keys(&:to_s)
+      end
+
+      def enriched_metadata(attributes)
+        metadata = stringify_keys(attributes["metadata"] || {})
+        %w[event_id source source_position changes type].each do |key|
+          metadata[key] = attributes[key] if attributes.key?(key)
+        end
+        metadata
+      end
+
+      def delete_values(attributes, data)
+        data if attributes["operation"].to_s == "delete"
+      end
+
+      def changed_values(attributes, data)
+        data unless attributes["operation"].to_s == "delete"
+      end
+
+      def parse_time(value)
+        return value if value.nil? || value.is_a?(Time)
+
+        Time.iso8601(value.to_s)
+      end
+    end
+  end
+end

data/lib/mammoth/replication_consumer.rb

@@ -20,7 +20,7 @@ module Mammoth
 
     # Consume normalized CDC work from the configured source.
     #
-    # @yieldparam event [Object] CDC::Core::ChangeEvent-compatible event
+    # @yieldparam event [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
     # @return [Integer] number of consumed events
     def start
       return enum_for(:start) unless block_given?
@@ -49,65 +49,33 @@ module Mammoth
 
     def flatten_cdc_work(work)
       return [] if work.nil?
-      return validate_transaction_envelope(work) if transaction_envelope?(work) && transaction_delivery?
-      return validate_events(work.events) if transaction_envelope?(work)
       return work.flat_map { |item| flatten_cdc_work(item) } if work.is_a?(Array)
-      return [synthetic_transaction_envelope(work)] if transaction_delivery?
+      return transaction_work(work) if work.is_a?(CDC::Core::TransactionEnvelope)
+      return event_work(work) if work.is_a?(CDC::Core::ChangeEvent)
 
-      validate_events([work])
+      raise ReplicationError, "CDC source yielded non-core work: #{work.class}"
     end
 
     def transaction_delivery?
       delivery_unit == :transaction
     end
 
-    def validate_transaction_envelope(envelope)
-      validate_events(envelope.events)
-      [envelope]
+    def transaction_work(envelope)
+      transaction_delivery? ? [envelope] : envelope.events
     end
 
-    def validate_events(events)
-      events.each { |event| validate_cdc_event!(event) }
+    def event_work(event)
+      transaction_delivery? ? [transaction_envelope(event)] : [event]
     end
 
-    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-    def synthetic_transaction_envelope(event)
-      validate_cdc_event!(event)
-
-      event_hash = event.to_h
-      SyntheticTransactionEnvelope.new(
-        [event],
-        event_hash["transaction_id"] || event_hash[:transaction_id] ||
-          event_hash["xid"] || event_hash[:xid] || event_hash["event_id"] || event_hash[:event_id],
-        event_hash["commit_lsn"] || event_hash[:commit_lsn] ||
-          event_hash["source_position"] || event_hash[:source_position],
-        event_hash["committed_at"] || event_hash[:committed_at] ||
-          event_hash["occurred_at"] || event_hash[:occurred_at],
-        event_hash["metadata"] || event_hash[:metadata] || {}
+    def transaction_envelope(event)
+      CDC::Core::TransactionEnvelope.new(
+        transaction_id: event.transaction_id || event.metadata[:event_id] || event.commit_lsn,
+        events: [event],
+        commit_lsn: event.commit_lsn,
+        committed_at: event.occurred_at,
+        metadata: event.metadata
       )
     end
-    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
-    def validate_cdc_event!(event)
-      return event if event.respond_to?(:to_h) && cdc_event_hash?(event.to_h)
-
-      raise ReplicationError, "CDC source yielded non-CDC work: #{event.class}"
-    end
-
-    def cdc_event_hash?(event_hash)
-      return false unless event_hash.respond_to?(:key?)
-
-      has_operation = event_hash.key?("operation") || event_hash.key?(:operation)
-      has_position = event_hash.key?("source_position") || event_hash.key?(:source_position) ||
-                     event_hash.key?("commit_lsn") || event_hash.key?(:commit_lsn)
-      has_operation && has_position
-    end
-
-    def transaction_envelope?(work)
-      work.respond_to?(:events) && work.respond_to?(:transaction_id)
-    end
-
-    SyntheticTransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
-    private_constant :SyntheticTransactionEnvelope
   end
 end

data/lib/mammoth/runtimes/batching_runtime.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Mammoth
+  module Runtimes
+    # Adds configured batch submission to a selected delivery runtime.
+    #
+    # Application streams one work item at a time into this runtime boundary.
+    # BatchingRuntime owns accumulation and submits complete or final partial
+    # batches through the selected adapter runtime's #process_many contract.
+    class BatchingRuntime
+      attr_reader :runtime, :batch_size, :buffer
+
+      # @param runtime [#process_many] selected delivery runtime
+      # @param batch_size [Integer] maximum work items submitted together
+      def initialize(runtime:, batch_size:)
+        @runtime = runtime
+        @batch_size = batch_size
+        @buffer = []
+      end
+
+      # Buffer one work item and submit a full batch when ready.
+      #
+      # @param work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
+      # @return [Array] processor results when a batch is submitted, otherwise an empty array
+      def process(work)
+        buffer << work
+        return [] if buffer.size < batch_size
+
+        flush
+      end
+
+      # Submit work immediately without changing the buffered stream.
+      #
+      # @param items [Array] work items
+      # @return [Array] processor results
+      def process_many(items)
+        runtime.process_many(items)
+      end
+
+      # Submit the final partial batch.
+      #
+      # @return [Array] processor results, or an empty array when no work is buffered
+      def flush
+        return [] if buffer.empty?
+
+        runtime.process_many(buffer.shift(buffer.size))
+      end
+
+      # Flush pending work and shut down the selected runtime when supported.
+      #
+      # @return [nil]
+      def shutdown
+        flush
+        runtime.shutdown if runtime.respond_to?(:shutdown)
+        nil
+      end
+    end
+  end
+end

data/lib/mammoth/runtimes/concurrent_adapter.rb

@@ -16,13 +16,15 @@ module Mammoth
         # @param concurrency [Integer] worker count
         # @param timeout [Numeric, nil] optional timeout
         # @param preserve_order [Boolean] preserve ordering when supported
+        # @param observer [CDC::Core::Observer] dispatch lifecycle observer
         # @return [Mammoth::ConcurrentDeliveryRuntime]
-        def build(processor:, concurrency:, timeout:, preserve_order:)
+        def build(processor:, concurrency:, timeout:, preserve_order:, observer: CDC::Core::Observer.new)
           ConcurrentDeliveryRuntime.new(
             processor: processor,
             concurrency: concurrency,
             timeout: timeout,
-            preserve_order: preserve_order
+            preserve_order: preserve_order,
+            observer: observer
           )
         end
       end

data/lib/mammoth/runtimes/inline_adapter.rb

@@ -4,20 +4,22 @@ module Mammoth
   module Runtimes
     # Inline runtime adapter that processes work in the caller thread.
     class InlineAdapter < Adapter
-      attr_reader :processor
+      attr_reader :processor, :observer
 
       # @param processor [#process] delivery processor
-      def initialize(processor:)
+      # @param observer [CDC::Core::Observer] dispatch lifecycle observer
+      def initialize(processor:, observer: CDC::Core::Observer.new)
         super()
         @processor = processor
+        @observer = observer
       end
 
       # Build an inline runtime.
       #
       # @param processor [#process] delivery processor
       # @return [Mammoth::Runtimes::InlineAdapter]
-      def self.build(processor:, **_options)
-        new(processor: processor)
+      def self.build(processor:, observer: CDC::Core::Observer.new, **_options)
+        new(processor: processor, observer: observer)
       end
 
       # @return [String] adapter type name
@@ -28,13 +30,25 @@ module Mammoth
       # @param items [Array<Object>] work units
       # @return [Array<Object>] processor results
       def process_many(items)
-        items.map { |item| processor.process(item) }
+        items.map do |item|
+          observer.dispatch_started(item)
+          processor.process(item).tap { |result| observe_result(result) }
+        end
       end
 
       # @return [nil]
       def shutdown
         nil
       end
+
+      private
+
+      def observe_result(result)
+        return observer.dispatch_succeeded(result) if result.success?
+        return observer.dispatch_failed(result) if result.failure?
+
+        observer.dispatch_skipped(result)
+      end
     end
   end
 end

data/lib/mammoth/runtimes/registry.rb

@@ -27,8 +27,9 @@ module Mammoth
         # @param name [String, Symbol] adapter name
         # @param options [Hash] adapter-specific build options
         # @return [Object] runtime adapter
-        def build(name, **options)
-          fetch(name).build(**options)
+        def build(name, batch_size: 1, **options)
+          runtime = fetch(name).build(**options)
+          BatchingRuntime.new(runtime:, batch_size:)
         end
 
         # @return [Array<String>] registered runtime adapter names

data/lib/mammoth/sources/postgres.rb

@@ -8,7 +8,7 @@ module Mammoth
     # Postgres realizes the CDC Ecosystem libraries for Mammoth's product
     # boundary. It composes pgoutput-client, pgoutput-parser,
     # pgoutput-decoder, and pgoutput-source-adapter into a single source that
-    # yields CDC::Core-shaped work to the delivery runtime.
+    # yields CDC::Core work to the delivery runtime.
     #
     # This class may mention pgoutput implementation details because it is the
     # concrete PostgreSQL source adapter used by Mammoth. The rest of Mammoth
@@ -45,21 +45,27 @@ module Mammoth
         @checkpoint_store = checkpoint_store
       end
 
-      # Stream CDC::Core-shaped work from PostgreSQL logical replication.
+      # Stream CDC::Core work from PostgreSQL logical replication.
       #
       # Calling this method starts the injected or configured pgoutput-client
       # runner. The runner owns the PostgreSQL replication connection and slot
-      # lifecycle; this class only composes the parser, decoder, and adapter
-      # libraries around the stream.
+      # lifecycle, while pgoutput-source-adapter owns transaction buffering and
+      # normalization into CDC::Core work items. This class only composes those
+      # layers and forwards transport source positions to the adapter.
       #
-      # @yieldparam work [Object] CDC::Core::ChangeEvent or TransactionEnvelope
+      # @yieldparam work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
       # @return [Enumerator, nil]
       # @raise [Mammoth::ReplicationError] when the source cannot stream CDC work
       def each(&block)
         return enum_for(:each) unless block_given?
 
-        effective_runner.start do |payload, metadata = nil|
-          process_payload(payload, metadata, &block)
+        normalizer = effective_adapter
+        unless normalizer.respond_to?(:each_normalized)
+          raise ReplicationError, "pgoutput source adapter must respond to #each_normalized"
+        end
+
+        normalizer.each_normalized(decoded_stream) do |work|
+          block.call(validate_core_work!(work))
         end
         nil
       rescue StandardError => e
@@ -70,41 +76,44 @@ module Mammoth
 
       private
 
+      def decoded_stream
+        Enumerator.new do |stream|
+          effective_runner.start do |payload, metadata = nil|
+            process_payload(payload, metadata) { |decoded| stream << stream_event(decoded, metadata) }
+          end
+        end
+      end
+
       def process_payload(payload, metadata, &block)
         parsed = parse_payload(payload)
         decoded = decode_message(parsed, metadata)
-        process_decoded(decoded, metadata, &block)
+        each_decoded(decoded, &block)
       end
 
-      # rubocop:disable Metrics/MethodLength
-      def process_decoded(decoded, metadata, &block)
+      def each_decoded(decoded, &block)
         return if decoded.nil?
 
         if decoded.is_a?(Array)
-          decoded.each { |item| process_decoded(item, metadata, &block) }
+          decoded.each { |item| each_decoded(item, &block) }
           return
         end
 
-        if begin_message?(decoded)
-          start_transaction_buffer(decoded)
-          return
-        end
+        block.call(decoded)
+      end
 
-        if commit_message?(decoded)
-          emit_transaction_buffer(decoded, metadata, &block)
-          return
-        end
+      def stream_event(decoded, metadata)
+        normalizer = effective_adapter
+        return decoded unless normalizer.respond_to?(:stream_event)
 
-        normalize_decoded(decoded).each do |work|
-          work = enrich_work_position(work, metadata, decoded)
-          if transaction_buffer_active?
-            Array(@transaction_events) << work
-          else
-            block.call(work)
-          end
-        end
+        normalizer.stream_event(decoded, source_position: source_position(metadata))
+      end
+
+      def validate_core_work!(work)
+        return work if work.is_a?(CDC::Core::ChangeEvent)
+        return work if work.is_a?(CDC::Core::TransactionEnvelope)
+
+        raise ReplicationError, "pgoutput source adapter yielded non-core work: #{work.class}"
       end
-      # rubocop:enable Metrics/MethodLength
 
       def parse_payload(payload)
         parser = effective_parser
@@ -127,102 +136,6 @@ module Mammoth
         raise ReplicationError, "pgoutput decoder must respond to #decode or #call"
       end
 
-      def normalize_decoded(decoded)
-        return [] if decoded.nil?
-        return decoded.flat_map { |item| normalize_decoded(item) } if decoded.is_a?(Array)
-
-        adapter = effective_adapter
-        result = if adapter.respond_to?(:normalize)
-                   adapter.normalize(decoded)
-                 elsif adapter.respond_to?(:call)
-                   adapter.call(decoded)
-                 else
-                   raise ReplicationError, "pgoutput source adapter must respond to #normalize or #call"
-                 end
-
-        result.is_a?(Array) ? result.compact : [result].compact
-      end
-
-      def begin_message?(decoded)
-        message_kind(decoded).include?("begin")
-      end
-
-      def commit_message?(decoded)
-        message_kind(decoded).include?("commit")
-      end
-
-      def message_kind(decoded)
-        (value_from(decoded, :message_type, :type, :kind) || decoded.class.name.to_s.split("::").last).to_s.downcase
-      end
-
-      def start_transaction_buffer(decoded)
-        @transaction_events = []
-        @transaction_id = value_from(decoded, :transaction_id, :xid, :final_lsn)
-        @transaction_metadata = value_hash(decoded, :metadata) || {}
-      end
-
-      def emit_transaction_buffer(decoded, metadata, &block)
-        return unless transaction_buffer_active?
-
-        block.call(
-          TransactionEnvelope.new(
-            @transaction_events,
-            transaction_id_for(decoded),
-            commit_lsn_for(decoded, metadata),
-            value_from(decoded, :commit_time, :committed_at, :timestamp),
-            @transaction_metadata
-          )
-        )
-      ensure
-        clear_transaction_buffer
-      end
-
-      def transaction_id_for(decoded)
-        value_from(decoded, :transaction_id, :xid) || @transaction_id || first_event_value(:transaction_id, :xid)
-      end
-
-      def commit_lsn_for(decoded, metadata)
-        value_from(decoded, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
-          value_from(metadata, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
-          first_event_value(:commit_lsn, :source_position, :lsn)
-      end
-
-      def transaction_buffer_active?
-        !@transaction_events.nil?
-      end
-
-      def clear_transaction_buffer
-        @transaction_events = nil
-        @transaction_id = nil
-        @transaction_metadata = nil
-      end
-
-      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      def enrich_work_position(work, metadata, decoded)
-        position = value_from(work, :source_position, :commit_lsn) ||
-                   value_from(metadata, :source_position, :commit_lsn, :lsn) ||
-                   value_from(decoded, :source_position, :commit_lsn, :lsn)
-        return work unless position
-
-        work_hash = work.respond_to?(:to_h) ? work.to_h : work
-        return work unless work_hash.is_a?(Hash)
-
-        key_style = work_hash.key?("operation") ? :string : :symbol
-        source_position_key = key_style == :string ? "source_position" : :source_position
-        commit_lsn_key = key_style == :string ? "commit_lsn" : :commit_lsn
-        return work if work_hash[source_position_key] || work_hash[commit_lsn_key]
-
-        work_hash.merge(source_position_key => position, commit_lsn_key => position)
-      end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
-      def first_event_value(*keys)
-        event = Array(@transaction_events).find do |event|
-          keys.any? { |key| value_from(event, key) }
-        end
-        value_from(event, *keys) if event
-      end
-
       def value_from(object, *keys)
         return nil if object.nil?
 
@@ -239,9 +152,8 @@ module Mammoth
         nil
       end
 
-      def value_hash(object, key)
-        value = value_from(object, key)
-        value.is_a?(Hash) ? value : nil
+      def source_position(metadata)
+        value_from(metadata, :source_position, :commit_lsn, :lsn, :wal_end_lsn, :end_lsn, :final_lsn)
       end
 
       def effective_runner
@@ -375,8 +287,6 @@ module Mammoth
       rescue LoadError => e
         raise ReplicationError, "#{gem_name} is required for PostgreSQL CDC source integration: #{e.message}"
       end
-      TransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
-      private_constant :TransactionEnvelope
     end
     # rubocop:enable Metrics/ClassLength
   end

data/lib/mammoth/status.rb

@@ -3,23 +3,23 @@
 module Mammoth
   # Builds and prints a boring operational status snapshot.
   class Status
-    attr_reader :config, :sqlite_store, :output
+    attr_reader :config, :state_adapter, :output
 
-    # Print status for a configuration and optional SQLite store.
+    # Print status for a configuration and optional operational-state adapter.
     #
     # @param config [Mammoth::Configuration] loaded configuration
-    # @param sqlite_store [Mammoth::SQLiteStore, nil] operational store
+    # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
     # @return [void]
-    def self.call(config, sqlite_store: nil, output: $stdout)
-      new(config, sqlite_store: sqlite_store, output: output).call
+    def self.call(config, state_adapter: nil, output: $stdout)
+      new(config, state_adapter: state_adapter, output: output).call
     end
 
     # @param config [Mammoth::Configuration] loaded configuration
-    # @param sqlite_store [Mammoth::SQLiteStore, nil] operational store
+    # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
     # @param output [#puts] output stream
-    def initialize(config, sqlite_store: nil, output: $stdout)
+    def initialize(config, state_adapter: nil, output: $stdout)
       @config = config
-      @sqlite_store = sqlite_store
+      @state_adapter = state_adapter
       @output = output
     end
 
@@ -28,13 +28,13 @@ module Mammoth
     # @return [void]
     def call
       status_lines.each { |line| output.puts(line) }
-      print_store_state if sqlite_store
+      print_store_state if state_adapter
     end
 
     private
 
     def status_lines
-      identity_lines + replication_lines + runtime_lines + destination_lines + ["SQLite: #{sqlite_path}"]
+      identity_lines + replication_lines + runtime_lines + destination_lines
     end
 
     def identity_lines
@@ -70,10 +70,6 @@ module Mammoth
       ]
     end
 
-    def sqlite_path
-      config.dig("sqlite", "path")
-    end
-
     def destination_names
       destinations = config.data["destinations"]
       return destinations.map { |destination| destination.fetch("name") } if destinations
@@ -90,10 +86,18 @@ module Mammoth
     end
 
     def print_store_state
-      store = sqlite_store.bootstrap!
-      output.puts "Tables: #{store.tables.join(", ")}"
-      output.puts "Checkpoints: #{CheckpointStore.new(store).count}"
-      output.puts "Dead Letters: #{DeadLetterStore.new(store).count}"
+      output.puts "Operational state ready: #{state_adapter.ready?}"
+      state_adapter.summary.each do |key, value|
+        output.puts "#{status_label(key)}: #{format_status_value(value)}"
+      end
+    end
+
+    def status_label(key)
+      key.to_s.split("_").map(&:capitalize).join(" ")
+    end
+
+    def format_status_value(value)
+      value.is_a?(Array) ? value.join(", ") : value
     end
   end
 end