mammoth 0.1.1 → 0.2.0

data/lib/mammoth/delivery_processor.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Adapter object used by CDC::Concurrent::ProcessorPool.
+  #
+  # The processor keeps cdc-concurrent integration narrow: cdc-concurrent owns
+  # I/O-heavy fan-out mechanics, while DeliveryWorker owns Mammoth relay
+  # semantics such as retries, dead letters, and checkpoint writes.
+  class DeliveryProcessor
+    @concurrent_safe = false
+
+    class << self
+      # Mark this processor as safe for CDC::Concurrent::ProcessorPool.
+      #
+      # DeliveryProcessor itself is intentionally stateless after initialization;
+      # per-delivery retry, dead-letter, and checkpoint behavior remains owned by
+      # the injected DeliveryWorker.
+      #
+      # @return [true]
+      def concurrent_safe!
+        @concurrent_safe = true
+      end
+
+      # @return [Boolean] true when this processor has explicitly opted in to
+      #   cdc-concurrent execution.
+      def concurrent_safe?
+        @concurrent_safe == true
+      end
+
+      alias concurrent_safe concurrent_safe?
+    end
+
+    concurrent_safe!
+
+    attr_reader :delivery_worker, :delivery_unit
+
+    # @param delivery_worker [Mammoth::DeliveryWorker] relay-aware delivery worker
+    # @param delivery_unit [String, Symbol] event or transaction
+    def initialize(delivery_worker:, delivery_unit: :event)
+      @delivery_worker = delivery_worker
+      @delivery_unit = delivery_unit.to_sym
+    end
+
+    # @return [Boolean] true when this processor instance is safe for concurrent execution.
+    def concurrent_safe?
+      self.class.concurrent_safe?
+    end
+
+    alias concurrent_safe concurrent_safe?
+
+    # Process one work item from CDC::Concurrent::ProcessorPool.
+    #
+    # @param work [Object] event or transaction envelope
+    # @return [Hash] delivery summary
+    def call(work)
+      process(work)
+    end
+
+    def process(work)
+      case delivery_unit
+      when :transaction
+        delivery_worker.deliver_transaction(work)
+      else
+        delivery_worker.deliver(work)
+      end
+    end
+  end
+end

data/lib/mammoth/delivery_worker.rb

@@ -11,12 +11,13 @@ module Mammoth
     # Default source name used when an event does not provide one.
     DEFAULT_SOURCE = "postgresql"
 
-    attr_reader :sink, :checkpoint_store, :dead_letter_store, :retry_schedule, :max_attempts, :sleeper, :source_name,
-                :slot_name, :publication_name
+    attr_reader :sink, :checkpoint_store, :dead_letter_store, :delivered_envelope_store, :retry_schedule, :max_attempts,
+                :sleeper, :source_name, :slot_name, :publication_name
 
     # @param sink [#deliver] destination sink
     # @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
     # @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
+    # @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore, nil] downstream delivery ledger
     # @param source_name [String] logical source name
     # @param slot_name [String] replication slot name
     # @param publication_name [String] publication name
@@ -24,10 +25,11 @@ module Mammoth
     # @param retry_schedule [Array<Integer>] retry wait schedule in seconds
     # @param sleeper [#call] sleep strategy, injectable for tests
     def initialize(sink:, checkpoint_store:, dead_letter_store:, source_name:, slot_name:, publication_name:,
-                   max_attempts:, retry_schedule:, sleeper: Kernel.method(:sleep))
+                   max_attempts:, retry_schedule:, delivered_envelope_store: nil, sleeper: Kernel.method(:sleep))
       @sink = sink
       @checkpoint_store = checkpoint_store
       @dead_letter_store = dead_letter_store
+      @delivered_envelope_store = delivered_envelope_store || DeliveredEnvelopeStore.new(checkpoint_store.sqlite_store)
       @source_name = source_name
       @slot_name = slot_name
       @publication_name = publication_name
@@ -58,30 +60,70 @@ module Mammoth
       )
     end
 
+    # Deliver a transaction envelope with retry, checkpoint, and DLQ handling.
+    #
+    # @param envelope [#events, #transaction_id] CDC transaction envelope
+    # @return [Hash] delivery summary
+    def deliver_transaction(envelope)
+      deliver_work(envelope, serializer: TransactionEnvelopeSerializer, delivery_method: :deliver_transaction)
+    end
+
     # Deliver an event with retry, checkpoint, and DLQ handling.
     #
     # @param event [Hash, #to_h] normalized event
     # @return [Hash] delivery summary
     def deliver(event)
+      deliver_work(event, serializer: EventSerializer, delivery_method: :deliver)
+    end
+
+    private
+
+    # rubocop:disable Metrics/MethodLength
+    def deliver_work(work, serializer:, delivery_method:)
       attempts = 0
+      payload = serializer.call(work)
+      delivery_unit = delivery_unit_for(delivery_method)
+      idempotency_key = idempotency_key_for(payload:, delivery_unit:)
+
+      if delivered_envelope_store.delivered?(idempotency_key)
+        checkpoint_payload(payload)
+        return {
+          status: "skipped",
+          duplicate: true,
+          idempotency_key: idempotency_key,
+          attempts: attempts,
+          destination: destination_name
+        }
+      end
 
       begin
         attempts += 1
-        result = sink.deliver(event)
-        checkpoint(event)
-        result.merge(attempts: attempts)
+        result = sink.public_send(delivery_method, work)
+        delivered_envelope_store.record!(
+          idempotency_key: idempotency_key,
+          source_name: source_name,
+          slot_name: slot_name,
+          destination_name: destination_name,
+          delivery_unit: delivery_unit.to_s,
+          transaction_id: payload["transaction_id"],
+          source_position: payload["source_position"]
+        )
+        checkpoint_payload(payload)
+        result.merge(attempts: attempts, idempotency_key: idempotency_key)
       rescue DeliveryError => e
-        return dead_letter(event, e, attempts) if attempts >= max_attempts
+        return dead_letter(work, e, attempts, serializer:) if attempts >= max_attempts
 
         wait_before_retry(attempts)
         retry
       end
     end
+    # rubocop:enable Metrics/MethodLength
 
-    private
+    def checkpoint(work, serializer:)
+      checkpoint_payload(serializer.call(work))
+    end
 
-    def checkpoint(event)
-      payload = EventSerializer.call(event)
+    def checkpoint_payload(payload)
       checkpoint_store.write(
         source_name: source_name,
         slot_name: slot_name,
@@ -90,12 +132,32 @@ module Mammoth
       )
     end
 
-    def dead_letter(event, error, attempts)
+    def idempotency_key_for(payload:, delivery_unit:)
+      [
+        source_name,
+        slot_name,
+        destination_name,
+        delivery_unit,
+        payload["transaction_id"] || payload["event_id"],
+        payload["source_position"]
+      ].compact.join(":")
+    end
+
+    def delivery_unit_for(delivery_method)
+      delivery_method == :deliver_transaction ? :transaction : :event
+    end
+
+    def destination_name
+      sink.respond_to?(:name) ? sink.name : sink.class.name
+    end
+
+    def dead_letter(event, error, attempts, serializer:)
       id = dead_letter_store.write(
         event: event,
         destination_name: sink.name,
         error: error,
-        retry_count: attempts
+        retry_count: attempts,
+        serializer: serializer
       )
       { status: "dead_lettered", dead_letter_id: id, attempts: attempts }
     end

data/lib/mammoth/replication_consumer.rb

@@ -5,14 +5,17 @@ module Mammoth
   #
   # ReplicationConsumer is intentionally upstream-agnostic. It does not know
   # which upstream system produced the work. Its
-  # only job is to consume CDC Ecosystem work, flatten CDC transaction
-  # envelopes, and yield individual change events to the delivery pipeline.
+  # job is to consume CDC Ecosystem work and yield either individual change
+  # events or transaction envelopes depending on Mammoth's configured delivery
+  # unit.
   class ReplicationConsumer
-    attr_reader :source
+    attr_reader :source, :delivery_unit
 
     # @param source [#each, nil] injectable CDC work stream
-    def initialize(source: nil)
+    # @param delivery_unit [String, Symbol] :event or :transaction
+    def initialize(source: nil, delivery_unit: :event)
       @source = source
+      @delivery_unit = delivery_unit.to_sym
     end
 
     # Consume normalized CDC work from the configured source.
@@ -46,16 +49,45 @@ module Mammoth
 
     def flatten_cdc_work(work)
       return [] if work.nil?
+      return validate_transaction_envelope(work) if transaction_envelope?(work) && transaction_delivery?
       return validate_events(work.events) if transaction_envelope?(work)
       return work.flat_map { |item| flatten_cdc_work(item) } if work.is_a?(Array)
+      return [synthetic_transaction_envelope(work)] if transaction_delivery?
 
       validate_events([work])
     end
 
+    def transaction_delivery?
+      delivery_unit == :transaction
+    end
+
+    def validate_transaction_envelope(envelope)
+      validate_events(envelope.events)
+      [envelope]
+    end
+
     def validate_events(events)
       events.each { |event| validate_cdc_event!(event) }
     end
 
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def synthetic_transaction_envelope(event)
+      validate_cdc_event!(event)
+
+      event_hash = event.to_h
+      SyntheticTransactionEnvelope.new(
+        [event],
+        event_hash["transaction_id"] || event_hash[:transaction_id] ||
+          event_hash["xid"] || event_hash[:xid] || event_hash["event_id"] || event_hash[:event_id],
+        event_hash["commit_lsn"] || event_hash[:commit_lsn] ||
+          event_hash["source_position"] || event_hash[:source_position],
+        event_hash["committed_at"] || event_hash[:committed_at] ||
+          event_hash["occurred_at"] || event_hash[:occurred_at],
+        event_hash["metadata"] || event_hash[:metadata] || {}
+      )
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
     def validate_cdc_event!(event)
       return event if event.respond_to?(:to_h) && cdc_event_hash?(event.to_h)
 
@@ -74,5 +106,8 @@ module Mammoth
     def transaction_envelope?(work)
       work.respond_to?(:events) && work.respond_to?(:transaction_id)
     end
+
+    SyntheticTransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
+    private_constant :SyntheticTransactionEnvelope
   end
 end

data/lib/mammoth/sources/postgres.rb

@@ -12,6 +12,7 @@ module Mammoth
     # This class may mention pgoutput implementation details because it is the
     # concrete PostgreSQL source adapter used by Mammoth. The rest of Mammoth
     # should remain source-agnostic and consume only the work yielded here.
+    # rubocop:disable Metrics/ClassLength
     class Postgres
       # @return [Mammoth::Configuration] loaded Mammoth configuration
       attr_reader :config
@@ -23,6 +24,8 @@ module Mammoth
       attr_reader :decoder
       # @return [Object, nil] injected CDC source adapter
       attr_reader :adapter
+      # @return [Mammoth::CheckpointStore, nil] checkpoint store used for restart resume
+      attr_reader :checkpoint_store
 
       # Build a PostgreSQL CDC source.
       #
@@ -31,12 +34,14 @@ module Mammoth
       # @param parser [Object, nil] injected pgoutput parser or relation tracker
       # @param decoder [Object, nil] injected pgoutput decoder
       # @param adapter [Object, nil] injected source adapter
-      def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil)
+      # @param checkpoint_store [Mammoth::CheckpointStore, nil] persisted checkpoints for restart resume
+      def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil, checkpoint_store: nil)
         @config = config
         @runner = runner
         @parser = parser
         @decoder = decoder
         @adapter = adapter
+        @checkpoint_store = checkpoint_store
       end
 
       # Stream CDC::Core-shaped work from PostgreSQL logical replication.
@@ -67,9 +72,39 @@ module Mammoth
       def process_payload(payload, metadata, &block)
         parsed = parse_payload(payload)
         decoded = decode_message(parsed, metadata)
-        normalize_decoded(decoded).each { |work| block.call(work) if work }
+        process_decoded(decoded, metadata, &block)
       end
 
+      # rubocop:disable Metrics/MethodLength
+      def process_decoded(decoded, metadata, &block)
+        return if decoded.nil?
+
+        if decoded.is_a?(Array)
+          decoded.each { |item| process_decoded(item, metadata, &block) }
+          return
+        end
+
+        if begin_message?(decoded)
+          start_transaction_buffer(decoded)
+          return
+        end
+
+        if commit_message?(decoded)
+          emit_transaction_buffer(decoded, metadata, &block)
+          return
+        end
+
+        normalize_decoded(decoded).each do |work|
+          work = enrich_work_position(work, metadata, decoded)
+          if transaction_buffer_active?
+            Array(@transaction_events) << work
+          else
+            block.call(work)
+          end
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+
       def parse_payload(payload)
         parser = effective_parser
         return parser.process(payload) if parser.respond_to?(:process)
@@ -107,20 +142,129 @@ module Mammoth
         result.is_a?(Array) ? result.compact : [result].compact
       end
 
+      def begin_message?(decoded)
+        message_kind(decoded).include?("begin")
+      end
+
+      def commit_message?(decoded)
+        message_kind(decoded).include?("commit")
+      end
+
+      def message_kind(decoded)
+        (value_from(decoded, :message_type, :type, :kind) || decoded.class.name.to_s.split("::").last).to_s.downcase
+      end
+
+      def start_transaction_buffer(decoded)
+        @transaction_events = []
+        @transaction_id = value_from(decoded, :transaction_id, :xid, :final_lsn)
+        @transaction_metadata = value_hash(decoded, :metadata) || {}
+      end
+
+      def emit_transaction_buffer(decoded, metadata, &block)
+        return unless transaction_buffer_active?
+
+        block.call(
+          TransactionEnvelope.new(
+            @transaction_events,
+            transaction_id_for(decoded),
+            commit_lsn_for(decoded, metadata),
+            value_from(decoded, :commit_time, :committed_at, :timestamp),
+            @transaction_metadata
+          )
+        )
+      ensure
+        clear_transaction_buffer
+      end
+
+      def transaction_id_for(decoded)
+        value_from(decoded, :transaction_id, :xid) || @transaction_id || first_event_value(:transaction_id, :xid)
+      end
+
+      def commit_lsn_for(decoded, metadata)
+        value_from(decoded, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
+          value_from(metadata, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
+          first_event_value(:commit_lsn, :source_position, :lsn)
+      end
+
+      def transaction_buffer_active?
+        !@transaction_events.nil?
+      end
+
+      def clear_transaction_buffer
+        @transaction_events = nil
+        @transaction_id = nil
+        @transaction_metadata = nil
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def enrich_work_position(work, metadata, decoded)
+        position = value_from(work, :source_position, :commit_lsn) ||
+                   value_from(metadata, :source_position, :commit_lsn, :lsn) ||
+                   value_from(decoded, :source_position, :commit_lsn, :lsn)
+        return work unless position
+
+        work_hash = work.respond_to?(:to_h) ? work.to_h : work
+        return work unless work_hash.is_a?(Hash)
+
+        key_style = work_hash.key?("operation") ? :string : :symbol
+        source_position_key = key_style == :string ? "source_position" : :source_position
+        commit_lsn_key = key_style == :string ? "commit_lsn" : :commit_lsn
+        return work if work_hash[source_position_key] || work_hash[commit_lsn_key]
+
+        work_hash.merge(source_position_key => position, commit_lsn_key => position)
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+      def first_event_value(*keys)
+        event = Array(@transaction_events).find do |event|
+          keys.any? { |key| value_from(event, key) }
+        end
+        value_from(event, *keys) if event
+      end
+
+      def value_from(object, *keys)
+        return nil if object.nil?
+
+        keys.each do |key|
+          return object.public_send(key) if object.respond_to?(key)
+
+          hash = object.respond_to?(:to_h) ? object.to_h : object
+          next unless hash.respond_to?(:key?)
+
+          return hash[key] if hash.key?(key)
+          return hash[key.to_s] if hash.key?(key.to_s)
+        end
+
+        nil
+      end
+
+      def value_hash(object, key)
+        value = value_from(object, key)
+        value.is_a?(Hash) ? value : nil
+      end
+
       def effective_runner
-        runner || (@effective_runner ||= build_runner)
+        runner || @effective_runner || begin
+          @effective_runner = build_runner
+        end
       end
 
       def effective_parser
-        parser || (@effective_parser ||= build_parser)
+        parser || @effective_parser || begin
+          @effective_parser = build_parser
+        end
       end
 
       def effective_decoder
-        decoder || (@effective_decoder ||= build_decoder)
+        decoder || @effective_decoder || begin
+          @effective_decoder = build_decoder
+        end
       end
 
       def effective_adapter
-        adapter || (@effective_adapter ||= build_adapter)
+        adapter || @effective_adapter || begin
+          @effective_adapter = build_adapter
+        end
       end
 
       def build_runner
@@ -152,15 +296,47 @@ module Mammoth
           database_url: database_url,
           slot_name: required_config("replication", "slot"),
           publication_names: required_publications,
-          start_lsn: config.dig("replication", "start_lsn"),
-          auto_create_slot: !config.dig("replication", "auto_create_slot").nil?,
-          temporary_slot: !config.dig("replication", "temporary_slot").nil?
+          start_lsn: replication_start_lsn,
+          auto_create_slot: config.dig("replication", "auto_create_slot") == true,
+          temporary_slot: config.dig("replication", "temporary_slot") == true
         }.tap do |options|
           feedback_interval = config.dig("replication", "feedback_interval")
           options[:feedback_interval] = feedback_interval unless feedback_interval.nil?
         end
       end
 
+      def replication_start_lsn
+        configured = config.dig("replication", "start_lsn")
+        return configured unless blank?(configured)
+
+        checkpoint_lsn
+      end
+
+      def checkpoint_lsn
+        return nil unless checkpoint_store
+
+        row = checkpoint_store.fetch(source_name: source_name, slot_name: required_config("replication", "slot"))
+        normalize_lsn(row&.fetch("last_lsn", nil))
+      end
+
+      def source_name
+        required_config("mammoth", "name")
+      end
+
+      def normalize_lsn(value)
+        return nil if blank?(value)
+
+        lsn = value.to_s
+        return lsn if lsn.include?("/")
+        return "0/#{lsn.to_i.to_s(16).upcase}" if lsn.match?(/\A\d+\z/)
+
+        lsn
+      end
+
+      def blank?(value)
+        value.nil? || value == ""
+      end
+
       def required_publications
         publications = required_config("replication", "publications")
         unless publications.is_a?(Array) && publications.any? &&
@@ -198,6 +374,9 @@ module Mammoth
       rescue LoadError => e
         raise ReplicationError, "#{gem_name} is required for PostgreSQL CDC source integration: #{e.message}"
       end
+      TransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
+      private_constant :TransactionEnvelope
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/mammoth/sql/__bootstrap__.sql

@@ -53,3 +53,27 @@ ON dead_letters(namespace, entity);
 
 CREATE INDEX IF NOT EXISTS idx_dead_letters_failed_at
 ON dead_letters(failed_at);
+
+
+CREATE TABLE IF NOT EXISTS delivered_envelopes (
+  id INTEGER PRIMARY KEY,
+  idempotency_key TEXT NOT NULL,
+  source_name TEXT NOT NULL,
+  slot_name TEXT NOT NULL,
+  destination_name TEXT NOT NULL,
+  delivery_unit TEXT NOT NULL,
+  transaction_id TEXT,
+  source_position TEXT,
+  delivered_at TEXT NOT NULL,
+
+  UNIQUE (idempotency_key)
+);
+
+CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source
+ON delivered_envelopes(source_name, slot_name);
+
+CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_destination
+ON delivered_envelopes(destination_name);
+
+CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source_position
+ON delivered_envelopes(source_position);

data/lib/mammoth/transaction_envelope_serializer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+
+module Mammoth
+  # Serializes CDC transaction envelopes into webhook payloads.
+  #
+  # Mammoth uses transaction envelopes as the safest delivery and checkpointing
+  # boundary for concurrent delivery. A transaction payload preserves the commit
+  # position and groups the row-level changes that belong to the same database
+  # transaction.
+  class TransactionEnvelopeSerializer
+    # Default payload type for transaction webhook delivery.
+    PAYLOAD_TYPE = "transaction.committed"
+
+    # Serialize a CDC::Core::TransactionEnvelope-like object into a Hash.
+    #
+    # @param envelope [#events, #transaction_id] transaction envelope
+    # @return [Hash] webhook-ready transaction payload
+    def self.call(envelope)
+      new(envelope).call
+    end
+
+    # @param envelope [#events, #transaction_id] transaction envelope
+    def initialize(envelope)
+      @envelope = envelope
+    end
+
+    # Return the webhook payload.
+    #
+    # @return [Hash] transaction webhook payload
+    def call
+      event_payloads = envelope.events.map { |event| EventSerializer.call(event) }
+      {
+        "event_id" => envelope_value("event_id") || SecureRandom.uuid,
+        "type" => PAYLOAD_TYPE,
+        "source" => first_event_value(event_payloads, "source") || EventSerializer::DEFAULT_SOURCE,
+        "transaction_id" => envelope.transaction_id,
+        "source_position" => source_position(event_payloads),
+        "commit_lsn" => source_position(event_payloads),
+        "committed_at" => committed_at,
+        "event_count" => event_payloads.length,
+        "events" => event_payloads,
+        "metadata" => envelope_value("metadata") || {}
+      }
+    end
+
+    # Return JSON representation of the transaction payload.
+    #
+    # @return [String] JSON representation
+    def to_json(*_args)
+      JSON.generate(call)
+    end
+
+    private
+
+    attr_reader :envelope
+
+    def envelope_hash
+      @envelope_hash ||= envelope.respond_to?(:to_h) ? stringify_keys(envelope.to_h) : {}
+    end
+
+    def stringify_keys(hash)
+      hash.to_h.transform_keys(&:to_s)
+    end
+
+    def envelope_value(key)
+      return envelope.public_send(key) if envelope.respond_to?(key)
+
+      envelope_hash[key]
+    end
+
+    def source_position(event_payloads)
+      envelope_value("source_position") || envelope_value("commit_lsn") ||
+        first_event_value(event_payloads.reverse, "source_position")
+    end
+
+    def committed_at
+      value = envelope_value("committed_at") || envelope_value("commit_time")
+      return value.iso8601 if value.respond_to?(:iso8601)
+
+      value || Time.now.utc.iso8601
+    end
+
+    def first_event_value(event_payloads, key)
+      event_payloads.find { |payload| payload[key] }&.fetch(key)
+    end
+  end
+end

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end