mammoth 0.8.0 → 1.0.0

data/lib/mammoth/replication_consumer.rb

@@ -20,14 +20,14 @@ module Mammoth
 
     # Consume normalized CDC work from the configured source.
     #
-    # @yieldparam event [Object] CDC::Core::ChangeEvent-compatible event
+    # @yieldparam event [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
     # @return [Integer] number of consumed events
     def start
       return enum_for(:start) unless block_given?
 
       count = 0
 
-      each_event do |event|
+      each_grouped_event do |event, _group_end|
         yield event
         count += 1
       end
@@ -35,11 +35,27 @@ module Mammoth
       count
     end
 
+    # Consume work with an explicit source-group boundary for safe progress.
+    #
+    # @yieldparam event [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
+    # @yieldparam group_end [Boolean] true for the final delivery in a source transaction
+    # @return [Integer] number of consumed work items
+    def start_with_boundaries
+      return enum_for(:start_with_boundaries) unless block_given?
+
+      count = 0
+      each_grouped_event do |event, group_end|
+        yield event, group_end
+        count += 1
+      end
+      count
+    end
+
     private
 
-    def each_event(&block)
+    def each_grouped_event(&block)
       effective_source.each do |work|
-        flatten_cdc_work(work).each(&block)
+        grouped_cdc_work(work).each { |event, group_end| block.call(event, group_end) }
       end
     end
 
@@ -47,67 +63,40 @@ module Mammoth
       source || raise(ReplicationError, "replication source is not configured")
     end
 
-    def flatten_cdc_work(work)
+    def grouped_cdc_work(work)
       return [] if work.nil?
-      return validate_transaction_envelope(work) if transaction_envelope?(work) && transaction_delivery?
-      return validate_events(work.events) if transaction_envelope?(work)
-      return work.flat_map { |item| flatten_cdc_work(item) } if work.is_a?(Array)
-      return [synthetic_transaction_envelope(work)] if transaction_delivery?
+      return work.flat_map { |item| grouped_cdc_work(item) } if work.is_a?(Array)
+      return grouped_transaction_work(work) if work.is_a?(CDC::Core::TransactionEnvelope)
+      return event_work(work).map { |item| [item, true] } if work.is_a?(CDC::Core::ChangeEvent)
+
+      raise ReplicationError, "CDC source yielded non-core work: #{work.class}"
+    end
 
-      validate_events([work])
+    def grouped_transaction_work(envelope)
+      items = transaction_work(envelope)
+      items.each_with_index.map { |item, index| [item, index == items.length - 1] }
     end
 
     def transaction_delivery?
       delivery_unit == :transaction
     end
 
-    def validate_transaction_envelope(envelope)
-      validate_events(envelope.events)
-      [envelope]
+    def transaction_work(envelope)
+      transaction_delivery? ? [envelope] : envelope.events
     end
 
-    def validate_events(events)
-      events.each { |event| validate_cdc_event!(event) }
+    def event_work(event)
+      transaction_delivery? ? [transaction_envelope(event)] : [event]
     end
 
-    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-    def synthetic_transaction_envelope(event)
-      validate_cdc_event!(event)
-
-      event_hash = event.to_h
-      SyntheticTransactionEnvelope.new(
-        [event],
-        event_hash["transaction_id"] || event_hash[:transaction_id] ||
-          event_hash["xid"] || event_hash[:xid] || event_hash["event_id"] || event_hash[:event_id],
-        event_hash["commit_lsn"] || event_hash[:commit_lsn] ||
-          event_hash["source_position"] || event_hash[:source_position],
-        event_hash["committed_at"] || event_hash[:committed_at] ||
-          event_hash["occurred_at"] || event_hash[:occurred_at],
-        event_hash["metadata"] || event_hash[:metadata] || {}
+    def transaction_envelope(event)
+      CDC::Core::TransactionEnvelope.new(
+        transaction_id: event.transaction_id || event.metadata[:event_id] || event.commit_lsn,
+        events: [event],
+        commit_lsn: event.commit_lsn,
+        committed_at: event.occurred_at,
+        metadata: event.metadata
       )
     end
-    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
-    def validate_cdc_event!(event)
-      return event if event.respond_to?(:to_h) && cdc_event_hash?(event.to_h)
-
-      raise ReplicationError, "CDC source yielded non-CDC work: #{event.class}"
-    end
-
-    def cdc_event_hash?(event_hash)
-      return false unless event_hash.respond_to?(:key?)
-
-      has_operation = event_hash.key?("operation") || event_hash.key?(:operation)
-      has_position = event_hash.key?("source_position") || event_hash.key?(:source_position) ||
-                     event_hash.key?("commit_lsn") || event_hash.key?(:commit_lsn)
-      has_operation && has_position
-    end
-
-    def transaction_envelope?(work)
-      work.respond_to?(:events) && work.respond_to?(:transaction_id)
-    end
-
-    SyntheticTransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
-    private_constant :SyntheticTransactionEnvelope
   end
 end

data/lib/mammoth/runtimes/batching_runtime.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Mammoth
+  module Runtimes
+    # Adds configured batch submission to a selected delivery runtime.
+    #
+    # Application streams one work item at a time into this runtime boundary.
+    # BatchingRuntime owns accumulation and submits complete or final partial
+    # batches through the selected adapter runtime's #process_many contract.
+    class BatchingRuntime
+      attr_reader :runtime, :batch_size, :buffer
+
+      # @param runtime [#process_many] selected delivery runtime
+      # @param batch_size [Integer] maximum work items submitted together
+      def initialize(runtime:, batch_size:)
+        @runtime = runtime
+        @batch_size = batch_size
+        @buffer = []
+      end
+
+      # Buffer one work item and submit a full batch when ready.
+      #
+      # @param work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
+      # @return [Array] processor results when a batch is submitted, otherwise an empty array
+      def process(work)
+        buffer << work
+        return [] if buffer.size < batch_size
+
+        flush
+      end
+
+      # Submit work immediately without changing the buffered stream.
+      #
+      # @param items [Array] work items
+      # @return [Array] processor results
+      def process_many(items)
+        runtime.process_many(items)
+      end
+
+      # Submit the final partial batch.
+      #
+      # @return [Array] processor results, or an empty array when no work is buffered
+      def flush
+        return [] if buffer.empty?
+
+        runtime.process_many(buffer.shift(buffer.size))
+      end
+
+      # Flush pending work and shut down the selected runtime when supported.
+      #
+      # @return [nil]
+      def shutdown
+        flush
+        runtime.shutdown if runtime.respond_to?(:shutdown)
+        nil
+      end
+    end
+  end
+end

data/lib/mammoth/runtimes/concurrent_adapter.rb

@@ -16,13 +16,15 @@ module Mammoth
         # @param concurrency [Integer] worker count
         # @param timeout [Numeric, nil] optional timeout
         # @param preserve_order [Boolean] preserve ordering when supported
+        # @param observer [CDC::Core::Observer] dispatch lifecycle observer
         # @return [Mammoth::ConcurrentDeliveryRuntime]
-        def build(processor:, concurrency:, timeout:, preserve_order:)
+        def build(processor:, concurrency:, timeout:, preserve_order:, observer: CDC::Core::Observer.new)
           ConcurrentDeliveryRuntime.new(
             processor: processor,
             concurrency: concurrency,
             timeout: timeout,
-            preserve_order: preserve_order
+            preserve_order: preserve_order,
+            observer: observer
           )
         end
       end

data/lib/mammoth/runtimes/inline_adapter.rb

@@ -4,20 +4,22 @@ module Mammoth
   module Runtimes
     # Inline runtime adapter that processes work in the caller thread.
     class InlineAdapter < Adapter
-      attr_reader :processor
+      attr_reader :processor, :observer
 
       # @param processor [#process] delivery processor
-      def initialize(processor:)
+      # @param observer [CDC::Core::Observer] dispatch lifecycle observer
+      def initialize(processor:, observer: CDC::Core::Observer.new)
         super()
         @processor = processor
+        @observer = observer
       end
 
       # Build an inline runtime.
       #
       # @param processor [#process] delivery processor
       # @return [Mammoth::Runtimes::InlineAdapter]
-      def self.build(processor:, **_options)
-        new(processor: processor)
+      def self.build(processor:, observer: CDC::Core::Observer.new, **_options)
+        new(processor: processor, observer: observer)
       end
 
       # @return [String] adapter type name
@@ -28,13 +30,25 @@ module Mammoth
       # @param items [Array<Object>] work units
       # @return [Array<Object>] processor results
       def process_many(items)
-        items.map { |item| processor.process(item) }
+        items.map do |item|
+          observer.dispatch_started(item)
+          processor.process(item).tap { |result| observe_result(result) }
+        end
       end
 
       # @return [nil]
       def shutdown
         nil
       end
+
+      private
+
+      def observe_result(result)
+        return observer.dispatch_succeeded(result) if result.success?
+        return observer.dispatch_failed(result) if result.failure?
+
+        observer.dispatch_skipped(result)
+      end
     end
   end
 end

data/lib/mammoth/runtimes/registry.rb

@@ -27,8 +27,9 @@ module Mammoth
         # @param name [String, Symbol] adapter name
         # @param options [Hash] adapter-specific build options
         # @return [Object] runtime adapter
-        def build(name, **options)
-          fetch(name).build(**options)
+        def build(name, batch_size: 1, **options)
+          runtime = fetch(name).build(**options)
+          BatchingRuntime.new(runtime:, batch_size:)
         end
 
         # @return [Array<String>] registered runtime adapter names

data/lib/mammoth/sources/postgres.rb

@@ -27,6 +27,8 @@ module Mammoth
       attr_reader :adapter
       # @return [Mammoth::CheckpointStore, nil] checkpoint store used for restart resume
       attr_reader :checkpoint_store
+      # @return [#inspect, nil] injected PostgreSQL publication inspector
+      attr_reader :publication_inspector
 
       # Build a PostgreSQL CDC source.
       #
@@ -36,13 +38,16 @@ module Mammoth
       # @param decoder [Object, nil] injected pgoutput decoder
       # @param adapter [Object, nil] injected source adapter
       # @param checkpoint_store [Mammoth::CheckpointStore, nil] persisted checkpoints for restart resume
-      def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil, checkpoint_store: nil)
+      # @param publication_inspector [#inspect, nil] injected publication metadata inspector
+      def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil, checkpoint_store: nil,
+                     publication_inspector: nil)
         @config = config
         @runner = runner
         @parser = parser
         @decoder = decoder
         @adapter = adapter
         @checkpoint_store = checkpoint_store
+        @publication_inspector = publication_inspector
       end
 
       # Stream CDC::Core work from PostgreSQL logical replication.
@@ -59,12 +64,16 @@ module Mammoth
       def each(&block)
         return enum_for(:each) unless block_given?
 
+        preflight_slot!
+        preflight_replica_identity!
         normalizer = effective_adapter
         unless normalizer.respond_to?(:each_normalized)
           raise ReplicationError, "pgoutput source adapter must respond to #each_normalized"
         end
 
-        normalizer.each_normalized(decoded_stream, &block)
+        normalizer.each_normalized(decoded_stream) do |work|
+          yield_with_progress_position(validate_core_work!(work), &block)
+        end
         nil
       rescue StandardError => e
         raise e if e.is_a?(ReplicationError)
@@ -72,11 +81,65 @@ module Mammoth
         raise ReplicationError, "PostgreSQL CDC source failed: #{e.message}"
       end
 
+      # Acknowledge a durably handled PostgreSQL WAL position.
+      #
+      # @param lsn [String, Integer] pgoutput-client compatible WAL position
+      # @return [Integer] normalized acknowledged WAL position
+      def acknowledge(lsn)
+        effective_runner.ack(lsn)
+      rescue StandardError => e
+        raise ReplicationError, "PostgreSQL WAL acknowledgement failed: #{e.message}"
+      end
+
+      # Resolve the acknowledgement-compatible transport position for work
+      # currently being yielded by this source.
+      #
+      # @param work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] yielded work
+      # @return [String, Integer, nil] pgoutput-client compatible WAL position
+      def progress_position_for(work)
+        return nil unless yielded_work_includes?(work)
+
+        @yielded_progress_position
+      end
+
+      # Inspect the configured slot for readiness and operator metrics.
+      #
+      # pgoutput-client owns catalog access. This method converts its snapshot
+      # into Mammoth's PostgreSQL-specific health policy without leaking
+      # transport-library types into the observability layer.
+      #
+      # @return [PostgresSlotHealth]
+      def slot_health
+        status = inspected_slot_status
+        return PostgresSlotHealth.missing(required_config("replication", "slot")) unless status
+
+        PostgresSlotHealth.new(
+          slot_name: value_from(status, :slot_name),
+          present: true,
+          active: value_from(status, :active) == true,
+          retained_wal_bytes: value_from(status, :retained_wal_bytes),
+          wal_status: value_from(status, :wal_status),
+          safe_wal_size: value_from(status, :safe_wal_size),
+          inactive_since: value_from(status, :inactive_since),
+          invalidation_reason: value_from(status, :invalidation_reason),
+          restart_lsn: value_from(status, :restart_lsn),
+          restart_lsn_bytes: metric_lsn(value_from(status, :restart_lsn)),
+          confirmed_flush_lsn: value_from(status, :confirmed_flush_lsn),
+          confirmed_flush_lsn_bytes: metric_lsn(value_from(status, :confirmed_flush_lsn)),
+          conflicting: value_from(status, :conflicting) == true
+        )
+      rescue StandardError => e
+        raise e if e.is_a?(ReplicationError)
+
+        raise ReplicationError, "PostgreSQL slot health inspection failed: #{e.message}"
+      end
+
       private
 
       def decoded_stream
         Enumerator.new do |stream|
           effective_runner.start do |payload, metadata = nil|
+            @latest_progress_position = acknowledgement_position(metadata)
             process_payload(payload, metadata) { |decoded| stream << stream_event(decoded, metadata) }
           end
         end
@@ -106,6 +169,22 @@ module Mammoth
         normalizer.stream_event(decoded, source_position: source_position(metadata))
       end
 
+      def yield_with_progress_position(work)
+        @yielded_work = work
+        @yielded_progress_position = @latest_progress_position
+        yield work
+      ensure
+        @yielded_work = nil
+        @yielded_progress_position = nil
+      end
+
+      def validate_core_work!(work)
+        return work if work.is_a?(CDC::Core::ChangeEvent)
+        return work if work.is_a?(CDC::Core::TransactionEnvelope)
+
+        raise ReplicationError, "pgoutput source adapter yielded non-core work: #{work.class}"
+      end
+
       def parse_payload(payload)
         parser = effective_parser
         return parser.process(payload) if parser.respond_to?(:process)
@@ -147,6 +226,22 @@ module Mammoth
         value_from(metadata, :source_position, :commit_lsn, :lsn, :wal_end_lsn, :end_lsn, :final_lsn)
       end
 
+      def acknowledgement_position(metadata)
+        value = value_from(metadata, :wal_end_lsn, :wal_end, :lsn)
+        return nil if value.nil?
+        return value if value.is_a?(Integer) && value >= 0
+        return value if value.is_a?(String) && value.match?(%r{\A[0-9A-F]+/[0-9A-F]+\z}i)
+
+        raise ReplicationError, "invalid PostgreSQL transport LSN for acknowledgement: #{value.inspect}"
+      end
+
+      def yielded_work_includes?(work)
+        return true if @yielded_work.equal?(work)
+        return false unless @yielded_work.is_a?(CDC::Core::TransactionEnvelope)
+
+        @yielded_work.events.any? { |event| event.equal?(work) }
+      end
+
       def effective_runner
         runner || @effective_runner || begin
           @effective_runner = build_runner
@@ -171,6 +266,12 @@ module Mammoth
         end
       end
 
+      def effective_publication_inspector
+        publication_inspector || @effective_publication_inspector || begin
+          @effective_publication_inspector = build_publication_inspector
+        end
+      end
+
       def build_runner
         require_optional!("pgoutput/client", "pgoutput-client")
 
@@ -192,7 +293,12 @@ module Mammoth
       def build_adapter
         require_optional!("pgoutput/source_adapter", "pgoutput-source-adapter")
 
-        Pgoutput::SourceAdapter::Cdc.new
+        resolver = Pgoutput::SourceAdapter::ReplicaIdentityResolver.new(replica_identity_relations)
+        Pgoutput::SourceAdapter::Cdc.new(primary_key_resolver: resolver)
+      end
+
+      def build_publication_inspector
+        PostgresPublicationInspector.new(database_url:)
       end
 
       def runner_options
@@ -201,7 +307,7 @@ module Mammoth
           slot_name: required_config("replication", "slot"),
           publication_names: required_publications,
           start_lsn: replication_start_lsn,
-          auto_create_slot: config.dig("replication", "auto_create_slot") == true,
+          auto_create_slot: safe_auto_create_slot?,
           temporary_slot: config.dig("replication", "temporary_slot") == true
         }.tap do |options|
           feedback_interval = config.dig("replication", "feedback_interval")
@@ -216,6 +322,157 @@ module Mammoth
         checkpoint_lsn
       end
 
+      def preflight_slot!
+        resume_lsn = replication_start_lsn
+        validate_temporary_slot_resume!(resume_lsn)
+        status = inspected_slot_status
+
+        if status.nil?
+          return if safe_auto_create_slot?
+
+          raise ReplicationError, missing_slot_message(resume_lsn)
+        end
+
+        validate_slot_identity!(status)
+        validate_slot_health!(status)
+        validate_slot_reachability!(status, resume_lsn) unless blank?(resume_lsn)
+        nil
+      rescue StandardError => e
+        raise e if e.is_a?(ReplicationError)
+
+        raise ReplicationError, "PostgreSQL slot preflight failed: #{e.message}"
+      end
+
+      def preflight_replica_identity!
+        invalid_tables = publication_tables.reject(&:identity_usable?)
+        return if invalid_tables.empty?
+
+        details = invalid_tables.map do |table|
+          "#{table.qualified_name} (actions=#{table.identity_actions.join("/")}, " \
+            "replica_identity=#{replica_identity_name(table.replica_identity)})"
+        end
+        raise ReplicationError,
+              "PostgreSQL replica identity preflight failed for #{details.join(", ")}. " \
+              "Add a primary key, select an eligible unique index with REPLICA IDENTITY USING INDEX, " \
+              "use REPLICA IDENTITY FULL, or remove UPDATE/DELETE from the publication."
+      rescue StandardError => e
+        raise e if e.is_a?(ReplicationError)
+
+        raise ReplicationError, "PostgreSQL replica identity preflight failed: #{e.message}"
+      end
+
+      def publication_tables
+        @publication_tables ||= effective_publication_inspector.inspect(required_publications).freeze
+      end
+
+      def replica_identity_relations
+        relations = {} # : Hash[Integer | [String, String], Array[String]]
+        publication_tables.each do |table|
+          next if table.replica_identity_columns.empty?
+
+          columns = table.replica_identity_columns
+          relations[table.relation_id] = columns
+          relations[[table.schema_name, table.table_name]] = columns
+        end
+        relations
+      end
+
+      def inspected_slot_status
+        client = effective_runner
+        unless client.respond_to?(:slot_status)
+          raise ReplicationError, "pgoutput-client 0.4+ with #slot_status is required for PostgreSQL slot inspection"
+        end
+
+        client.slot_status
+      end
+
+      def validate_temporary_slot_resume!(resume_lsn)
+        return if blank?(resume_lsn)
+        return unless config.dig("replication", "temporary_slot") == true
+
+        raise ReplicationError, "cannot resume durable PostgreSQL checkpoint #{resume_lsn} with a temporary slot"
+      end
+
+      def validate_slot_identity!(status)
+        slot_name = required_config("replication", "slot")
+        unless value_from(status, :slot_name) == slot_name
+          raise ReplicationError, "PostgreSQL slot preflight returned the wrong slot for #{slot_name}"
+        end
+        unless value_from(status, :slot_type) == "logical" && value_from(status, :plugin) == "pgoutput"
+          raise ReplicationError, "PostgreSQL slot #{slot_name} must be a logical pgoutput slot"
+        end
+
+        database = required_config("postgres", "database")
+        return if value_from(status, :database) == database
+
+        raise ReplicationError, "PostgreSQL slot #{slot_name} belongs to a different database"
+      end
+
+      def validate_slot_health!(status)
+        slot_name = required_config("replication", "slot")
+        raise ReplicationError, "PostgreSQL slot #{slot_name} is already active" if value_from(status, :active) == true
+
+        wal_status = value_from(status, :wal_status)
+        if %w[lost unreserved].include?(wal_status)
+          raise ReplicationError, "PostgreSQL slot #{slot_name} cannot retain required WAL (wal_status=#{wal_status})"
+        end
+
+        invalidation_reason = value_from(status, :invalidation_reason)
+        if value_from(status, :conflicting) == true || !blank?(invalidation_reason)
+          raise ReplicationError,
+                "PostgreSQL slot #{slot_name} is invalidated#{": #{invalidation_reason}" unless blank?(invalidation_reason)}"
+        end
+
+        return unless blank?(value_from(status, :restart_lsn))
+
+        raise ReplicationError, "PostgreSQL slot #{slot_name} has no reachable restart LSN"
+      end
+
+      def validate_slot_reachability!(status, resume_lsn)
+        requested = parse_transport_lsn(resume_lsn, "resume checkpoint")
+        %i[restart_lsn confirmed_flush_lsn].each do |field|
+          boundary = value_from(status, field)
+          next if blank?(boundary)
+          next unless requested < parse_transport_lsn(boundary, field.to_s)
+
+          raise ReplicationError,
+                "PostgreSQL slot cannot serve checkpoint #{resume_lsn}; #{field}=#{boundary} has already advanced past it"
+        end
+      end
+
+      def parse_transport_lsn(value, label)
+        require_optional!("pgoutput/client", "pgoutput-client")
+        Pgoutput::Client::LSN.parse(value)
+      rescue StandardError => e
+        raise ReplicationError, "invalid PostgreSQL #{label} LSN #{value.inspect}: #{e.message}"
+      end
+
+      def metric_lsn(value)
+        return nil if blank?(value)
+
+        parse_transport_lsn(value, "slot metric")
+      end
+
+      def replica_identity_name(value)
+        {
+          "d" => "default",
+          "n" => "nothing",
+          "f" => "full",
+          "i" => "index"
+        }.fetch(value, value)
+      end
+
+      def safe_auto_create_slot?
+        config.dig("replication", "auto_create_slot") == true && blank?(replication_start_lsn)
+      end
+
+      def missing_slot_message(resume_lsn)
+        slot_name = required_config("replication", "slot")
+        return "PostgreSQL slot #{slot_name} is missing and auto_create_slot is disabled" if blank?(resume_lsn)
+
+        "PostgreSQL slot #{slot_name} is missing; refusing to recreate it while checkpoint #{resume_lsn} requires continuity"
+      end
+
       def checkpoint_lsn
         return nil unless checkpoint_store