mammoth 0.1.1 → 0.3.0

data/lib/mammoth/dead_letter_commands.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Mammoth
+  # Operator commands for inspecting and replaying dead letters.
+  # rubocop:disable Metrics/ClassLength
+  class DeadLetterCommands
+    # Internal replay envelope used for transaction dead-letter recovery.
+    DEAD_LETTER_TRANSACTION_ENVELOPE = Data.define(
+      :events,
+      :transaction_id,
+      :source_position,
+      :commit_lsn,
+      :committed_at,
+      :metadata
+    )
+
+    attr_reader :argv
+
+    # @param argv [Array<String>] command line arguments
+    def self.call(argv)
+      new(argv).call
+    end
+
+    # @param argv [Array<String>] command line arguments
+    def initialize(argv)
+      @argv = argv
+    end
+
+    # Dispatch the nested dead-letter subcommand.
+    #
+    # @return [Integer] process status code
+    def call
+      case command
+      when "list" then list
+      when "show" then show
+      when "replay" then replay
+      else
+        raise ConfigurationError, "dead-letters subcommand required\n#{CLI::USAGE}"
+      end
+    rescue Mammoth::Error => e
+      warn e.message
+      1
+    end
+
+    private
+
+    def command
+      argv.fetch(1, nil)
+    end
+
+    def config_path
+      argv.fetch(2, nil)
+    end
+
+    def list
+      rows = dead_letter_store.rows(status: list_options.fetch(:status), limit: list_options.fetch(:limit))
+      puts list_header
+      rows.each { |row| puts list_row(row) }
+      0
+    end
+
+    def show
+      row = fetch_dead_letter!(dead_letter_id)
+      puts JSON.pretty_generate(show_payload(row))
+      0
+    end
+
+    def replay
+      rows = replay_rows
+      raise ConfigurationError, "no dead letters found to replay" if rows.empty?
+
+      rows.each do |row|
+        result = replay_row(row)
+        dead_letter_store.resolve(row.fetch("id")) unless result.fetch(:status) == "dead_lettered"
+        puts replay_message(row, result)
+      end
+      0
+    end
+
+    def load_config
+      raise ConfigurationError, "configuration path required\n#{CLI::USAGE}" unless config_path
+
+      Configuration.load(config_path)
+    end
+
+    def dead_letter_store
+      @dead_letter_store ||= DeadLetterStore.new(SQLiteStore.connect(load_config.dig("sqlite", "path")).bootstrap!)
+    end
+
+    def worker
+      @worker ||= Application.new(load_config).delivery_worker
+    end
+
+    def dead_letter_id
+      raw_id = argv.fetch(3, nil)
+      raise ConfigurationError, "dead letter id required\n#{CLI::USAGE}" unless raw_id
+
+      Integer(raw_id)
+    rescue ArgumentError
+      raise ConfigurationError, "dead letter id must be an integer"
+    end
+
+    # Parse dead-letter list filters and pagination.
+    #
+    # @return [Hash] list options
+    def list_options
+      options = { status: "pending", limit: 100 }
+      index = 0
+      args = argv.drop(3)
+
+      # rubocop:disable Style/WhileUntilModifier
+      while index < args.length
+        index = parse_list_option(args, index, options)
+      end
+      # rubocop:enable Style/WhileUntilModifier
+
+      options
+    end
+
+    def parse_list_option(args, index, options)
+      case args.fetch(index)
+      when "--status"
+        options[:status] = args.fetch(index + 1)
+        index + 2
+      when "--limit"
+        options[:limit] = Integer(args.fetch(index + 1))
+        index + 2
+      else
+        raise ConfigurationError, "unknown option #{args[index]}\n#{CLI::USAGE}" if args[index].start_with?("--")
+
+        raise ConfigurationError, "unexpected argument #{args[index]}\n#{CLI::USAGE}"
+      end
+    rescue ArgumentError, IndexError
+      raise ConfigurationError, "dead letter limit must be an integer"
+    end
+
+    # Resolve the rows that should be replayed.
+    #
+    # @return [Array<Hash>] replay rows
+    def replay_rows
+      ids = argv.drop(3)
+      return dead_letter_store.pending if ids.empty?
+
+      ids.map do |raw_id|
+        id = Integer(raw_id)
+        row = dead_letter_store.fetch(id)
+        raise ConfigurationError, "dead letter not found: #{id}" unless row
+
+        row
+      rescue ArgumentError
+        raise ConfigurationError, "dead letter id must be an integer"
+      end
+    end
+
+    def fetch_dead_letter!(id)
+      row = dead_letter_store.fetch(id)
+      raise ConfigurationError, "dead letter not found: #{id}" unless row
+
+      row
+    end
+
+    def replay_row(row)
+      payload = JSON.parse(row.fetch("payload_json"))
+      transaction_payload?(payload) ? worker.deliver_transaction(transaction_envelope(payload)) : worker.deliver(payload)
+    end
+
+    def transaction_payload?(payload)
+      payload.fetch("type", nil) == TransactionEnvelopeSerializer::PAYLOAD_TYPE
+    end
+
+    def transaction_envelope(payload)
+      DEAD_LETTER_TRANSACTION_ENVELOPE.new(
+        payload.fetch("events"),
+        payload.fetch("transaction_id"),
+        payload["source_position"],
+        payload["commit_lsn"],
+        payload["committed_at"],
+        payload["metadata"] || {}
+      )
+    end
+
+    def show_payload(row)
+      row.merge("payload" => JSON.parse(row.fetch("payload_json")))
+    end
+
+    def list_header
+      format(
+        "%<id>-4s  %<status>-10s  %<destination>-18s  %<retries>-8s  %<failed_at>-19s  %<event>s",
+        id: "ID",
+        status: "STATUS",
+        destination: "DESTINATION",
+        retries: "RETRIES",
+        failed_at: "FAILED_AT",
+        event: "EVENT"
+      )
+    end
+
+    def list_row(row)
+      format(
+        "%<id>-4s  %<status>-10s  %<destination>-18s  %<retries>-8s  %<failed_at>-19s  %<event>s",
+        id: row.fetch("id"),
+        status: row.fetch("status"),
+        destination: row.fetch("destination_name"),
+        retries: row.fetch("retry_count"),
+        failed_at: row.fetch("failed_at"),
+        event: row.fetch("event_id")
+      )
+    end
+
+    def replay_message(row, result)
+      "Dead letter #{row.fetch("id")}: #{result.fetch(:status)}"
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/mammoth/dead_letter_store.rb

@@ -20,9 +20,9 @@ module Mammoth
     # @param error [Exception, nil] delivery failure
     # @param retry_count [Integer] number of delivery attempts
     # @return [Integer] inserted dead letter id
-    def write(event:, destination_name:, error: nil, retry_count: 0) # rubocop:disable Metrics/MethodLength
+    def write(event:, destination_name:, error: nil, retry_count: 0, serializer: EventSerializer) # rubocop:disable Metrics/MethodLength
       now = Time.now.utc.iso8601
-      payload = EventSerializer.call(event)
+      payload = serializer.call(event)
       database.execute(
         <<~SQL,
           INSERT INTO dead_letters(
@@ -46,7 +46,7 @@ module Mammoth
           payload.fetch("event_id"),
           payload.fetch("source"),
           destination_name,
-          payload.fetch("operation"),
+          payload["operation"] || payload["type"],
           payload["namespace"],
           payload["entity"],
           payload["source_position"],
@@ -66,10 +66,31 @@ module Mammoth
     # @param limit [Integer] maximum number of rows
     # @return [Array<Hash>] pending dead letter rows
     def pending(limit: 100)
-      database.execute(
-        "SELECT * FROM dead_letters WHERE status = ? ORDER BY failed_at ASC LIMIT ?",
-        ["pending", limit]
-      )
+      rows(status: "pending", limit: limit)
+    end
+
+    # Fetch dead letters, optionally filtered by status.
+    #
+    # @param status [String, nil] optional dead-letter status filter
+    # @param limit [Integer] maximum number of rows
+    # @return [Array<Hash>] dead letter rows
+    def rows(status: nil, limit: 100)
+      if status && status != "all"
+        return database.execute(
+          "SELECT * FROM dead_letters WHERE status = ? ORDER BY failed_at ASC LIMIT ?",
+          [status, limit]
+        )
+      end
+
+      database.execute("SELECT * FROM dead_letters ORDER BY failed_at ASC LIMIT ?", [limit])
+    end
+
+    # Fetch one dead letter by id.
+    #
+    # @param id [Integer] dead letter id
+    # @return [Hash, nil] dead letter row
+    def fetch(id)
+      database.get_first_row("SELECT * FROM dead_letters WHERE id = ?", [id])
     end
 
     # Count dead letters by status.

data/lib/mammoth/delivered_envelope_store.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Mammoth
+  # SQLite-backed ledger of downstream deliveries.
+  #
+  # The PostgreSQL replication boundary is at-least-once. Mammoth therefore
+  # keeps a small delivery ledger so a transaction replayed by the upstream
+  # replication source after restart does not have to be delivered downstream again.
+  class DeliveredEnvelopeStore
+    # SQLite schema used to bootstrap the delivered-envelope ledger.
+    SCHEMA = <<~SQL
+      CREATE TABLE IF NOT EXISTS delivered_envelopes (
+        id INTEGER PRIMARY KEY,
+        idempotency_key TEXT NOT NULL,
+        source_name TEXT NOT NULL,
+        slot_name TEXT NOT NULL,
+        destination_name TEXT NOT NULL,
+        delivery_unit TEXT NOT NULL,
+        transaction_id TEXT,
+        source_position TEXT,
+        delivered_at TEXT NOT NULL,
+
+        UNIQUE (idempotency_key)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source
+      ON delivered_envelopes(source_name, slot_name);
+
+      CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_destination
+      ON delivered_envelopes(destination_name);
+
+      CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source_position
+      ON delivered_envelopes(source_position);
+    SQL
+
+    attr_reader :sqlite_store
+
+    def initialize(sqlite_store)
+      @sqlite_store = sqlite_store
+      ensure_schema!
+    end
+
+    def delivered?(idempotency_key)
+      !database.execute(
+        "SELECT 1 FROM delivered_envelopes WHERE idempotency_key = ? LIMIT 1",
+        [idempotency_key]
+      ).empty?
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def record!(idempotency_key:, source_name:, slot_name:, destination_name:, delivery_unit:, transaction_id:,
+                source_position:)
+      database.execute(
+        <<~SQL,
+          INSERT OR IGNORE INTO delivered_envelopes(
+            idempotency_key, source_name, slot_name, destination_name, delivery_unit,
+            transaction_id, source_position, delivered_at
+          )
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        SQL
+        [
+          idempotency_key,
+          source_name,
+          slot_name,
+          destination_name,
+          delivery_unit,
+          transaction_id,
+          source_position,
+          Time.now.utc.iso8601
+        ]
+      )
+      database.get_first_row(
+        "SELECT * FROM delivered_envelopes WHERE idempotency_key = ? LIMIT 1",
+        [idempotency_key]
+      )
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def all
+      database.execute("SELECT * FROM delivered_envelopes ORDER BY id")
+    end
+
+    # Count delivered envelopes.
+    #
+    # @return [Integer] delivered envelope count
+    def count
+      database.get_first_value("SELECT COUNT(*) FROM delivered_envelopes")
+    end
+
+    private
+
+    def ensure_schema!
+      database.execute_batch(SCHEMA)
+    end
+
+    def database
+      sqlite_store.bootstrap!.database
+    end
+  end
+end

data/lib/mammoth/delivery_processor.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Adapter object used by CDC::Concurrent::ProcessorPool.
+  #
+  # The processor keeps cdc-concurrent integration narrow: cdc-concurrent owns
+  # I/O-heavy fan-out mechanics, while DeliveryWorker owns Mammoth relay
+  # semantics such as retries, dead letters, and checkpoint writes.
+  class DeliveryProcessor
+    @concurrent_safe = false
+
+    class << self
+      # Mark this processor as safe for CDC::Concurrent::ProcessorPool.
+      #
+      # DeliveryProcessor itself is intentionally stateless after initialization;
+      # per-delivery retry, dead-letter, and checkpoint behavior remains owned by
+      # the injected DeliveryWorker.
+      #
+      # @return [true]
+      def concurrent_safe!
+        @concurrent_safe = true
+      end
+
+      # @return [Boolean] true when this processor has explicitly opted in to
+      #   cdc-concurrent execution.
+      def concurrent_safe?
+        @concurrent_safe == true
+      end
+
+      alias concurrent_safe concurrent_safe?
+    end
+
+    concurrent_safe!
+
+    attr_reader :delivery_worker, :delivery_unit
+
+    # @param delivery_worker [Mammoth::DeliveryWorker] relay-aware delivery worker
+    # @param delivery_unit [String, Symbol] event or transaction
+    def initialize(delivery_worker:, delivery_unit: :event)
+      @delivery_worker = delivery_worker
+      @delivery_unit = delivery_unit.to_sym
+    end
+
+    # @return [Boolean] true when this processor instance is safe for concurrent execution.
+    def concurrent_safe?
+      self.class.concurrent_safe?
+    end
+
+    alias concurrent_safe concurrent_safe?
+
+    # Process one work item from CDC::Concurrent::ProcessorPool.
+    #
+    # @param work [Object] event or transaction envelope
+    # @return [Hash] delivery summary
+    def call(work)
+      process(work)
+    end
+
+    # Process one work item using the configured delivery unit.
+    #
+    # @param work [Object] event or transaction envelope
+    # @return [Hash] delivery summary
+    def process(work)
+      case delivery_unit
+      when :transaction
+        delivery_worker.deliver_transaction(work)
+      else
+        delivery_worker.deliver(work)
+      end
+    end
+  end
+end

data/lib/mammoth/delivery_worker.rb

@@ -11,12 +11,13 @@ module Mammoth
     # Default source name used when an event does not provide one.
     DEFAULT_SOURCE = "postgresql"
 
-    attr_reader :sink, :checkpoint_store, :dead_letter_store, :retry_schedule, :max_attempts, :sleeper, :source_name,
-                :slot_name, :publication_name
+    attr_reader :sink, :checkpoint_store, :dead_letter_store, :delivered_envelope_store, :retry_schedule, :max_attempts,
+                :sleeper, :source_name, :slot_name, :publication_name
 
     # @param sink [#deliver] destination sink
     # @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
     # @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
+    # @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore, nil] downstream delivery ledger
     # @param source_name [String] logical source name
     # @param slot_name [String] replication slot name
     # @param publication_name [String] publication name
@@ -24,10 +25,11 @@ module Mammoth
     # @param retry_schedule [Array<Integer>] retry wait schedule in seconds
     # @param sleeper [#call] sleep strategy, injectable for tests
     def initialize(sink:, checkpoint_store:, dead_letter_store:, source_name:, slot_name:, publication_name:,
-                   max_attempts:, retry_schedule:, sleeper: Kernel.method(:sleep))
+                   max_attempts:, retry_schedule:, delivered_envelope_store: nil, sleeper: Kernel.method(:sleep))
       @sink = sink
       @checkpoint_store = checkpoint_store
       @dead_letter_store = dead_letter_store
+      @delivered_envelope_store = delivered_envelope_store || DeliveredEnvelopeStore.new(checkpoint_store.sqlite_store)
       @source_name = source_name
       @slot_name = slot_name
       @publication_name = publication_name
@@ -58,30 +60,70 @@ module Mammoth
       )
     end
 
+    # Deliver a transaction envelope with retry, checkpoint, and DLQ handling.
+    #
+    # @param envelope [#events, #transaction_id] CDC transaction envelope
+    # @return [Hash] delivery summary
+    def deliver_transaction(envelope)
+      deliver_work(envelope, serializer: TransactionEnvelopeSerializer, delivery_method: :deliver_transaction)
+    end
+
     # Deliver an event with retry, checkpoint, and DLQ handling.
     #
     # @param event [Hash, #to_h] normalized event
     # @return [Hash] delivery summary
     def deliver(event)
+      deliver_work(event, serializer: EventSerializer, delivery_method: :deliver)
+    end
+
+    private
+
+    # rubocop:disable Metrics/MethodLength
+    def deliver_work(work, serializer:, delivery_method:)
       attempts = 0
+      payload = serializer.call(work)
+      delivery_unit = delivery_unit_for(delivery_method)
+      idempotency_key = idempotency_key_for(payload:, delivery_unit:)
+
+      if delivered_envelope_store.delivered?(idempotency_key)
+        checkpoint_payload(payload)
+        return {
+          status: "skipped",
+          duplicate: true,
+          idempotency_key: idempotency_key,
+          attempts: attempts,
+          destination: destination_name
+        }
+      end
 
       begin
         attempts += 1
-        result = sink.deliver(event)
-        checkpoint(event)
-        result.merge(attempts: attempts)
+        result = sink.public_send(delivery_method, work)
+        delivered_envelope_store.record!(
+          idempotency_key: idempotency_key,
+          source_name: source_name,
+          slot_name: slot_name,
+          destination_name: destination_name,
+          delivery_unit: delivery_unit.to_s,
+          transaction_id: payload["transaction_id"],
+          source_position: payload["source_position"]
+        )
+        checkpoint_payload(payload)
+        result.merge(attempts: attempts, idempotency_key: idempotency_key)
       rescue DeliveryError => e
-        return dead_letter(event, e, attempts) if attempts >= max_attempts
+        return dead_letter(work, e, attempts, serializer:) if attempts >= max_attempts
 
         wait_before_retry(attempts)
         retry
       end
     end
+    # rubocop:enable Metrics/MethodLength
 
-    private
+    def checkpoint(work, serializer:)
+      checkpoint_payload(serializer.call(work))
+    end
 
-    def checkpoint(event)
-      payload = EventSerializer.call(event)
+    def checkpoint_payload(payload)
       checkpoint_store.write(
         source_name: source_name,
         slot_name: slot_name,
@@ -90,12 +132,32 @@ module Mammoth
       )
     end
 
-    def dead_letter(event, error, attempts)
+    def idempotency_key_for(payload:, delivery_unit:)
+      [
+        source_name,
+        slot_name,
+        destination_name,
+        delivery_unit,
+        payload["transaction_id"] || payload["event_id"],
+        payload["source_position"]
+      ].compact.join(":")
+    end
+
+    def delivery_unit_for(delivery_method)
+      delivery_method == :deliver_transaction ? :transaction : :event
+    end
+
+    def destination_name
+      sink.respond_to?(:name) ? sink.name : sink.class.name
+    end
+
+    def dead_letter(event, error, attempts, serializer:)
       id = dead_letter_store.write(
         event: event,
         destination_name: sink.name,
         error: error,
-        retry_count: attempts
+        retry_count: attempts,
+        serializer: serializer
       )
       { status: "dead_lettered", dead_letter_id: id, attempts: attempts }
     end

data/lib/mammoth/replication_consumer.rb

@@ -5,14 +5,17 @@ module Mammoth
   #
   # ReplicationConsumer is intentionally upstream-agnostic. It does not know
   # which upstream system produced the work. Its
-  # only job is to consume CDC Ecosystem work, flatten CDC transaction
-  # envelopes, and yield individual change events to the delivery pipeline.
+  # job is to consume CDC Ecosystem work and yield either individual change
+  # events or transaction envelopes depending on Mammoth's configured delivery
+  # unit.
   class ReplicationConsumer
-    attr_reader :source
+    attr_reader :source, :delivery_unit
 
     # @param source [#each, nil] injectable CDC work stream
-    def initialize(source: nil)
+    # @param delivery_unit [String, Symbol] :event or :transaction
+    def initialize(source: nil, delivery_unit: :event)
       @source = source
+      @delivery_unit = delivery_unit.to_sym
     end
 
     # Consume normalized CDC work from the configured source.
@@ -46,16 +49,45 @@ module Mammoth
 
     def flatten_cdc_work(work)
       return [] if work.nil?
+      return validate_transaction_envelope(work) if transaction_envelope?(work) && transaction_delivery?
       return validate_events(work.events) if transaction_envelope?(work)
       return work.flat_map { |item| flatten_cdc_work(item) } if work.is_a?(Array)
+      return [synthetic_transaction_envelope(work)] if transaction_delivery?
 
       validate_events([work])
     end
 
+    def transaction_delivery?
+      delivery_unit == :transaction
+    end
+
+    def validate_transaction_envelope(envelope)
+      validate_events(envelope.events)
+      [envelope]
+    end
+
     def validate_events(events)
       events.each { |event| validate_cdc_event!(event) }
     end
 
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def synthetic_transaction_envelope(event)
+      validate_cdc_event!(event)
+
+      event_hash = event.to_h
+      SyntheticTransactionEnvelope.new(
+        [event],
+        event_hash["transaction_id"] || event_hash[:transaction_id] ||
+          event_hash["xid"] || event_hash[:xid] || event_hash["event_id"] || event_hash[:event_id],
+        event_hash["commit_lsn"] || event_hash[:commit_lsn] ||
+          event_hash["source_position"] || event_hash[:source_position],
+        event_hash["committed_at"] || event_hash[:committed_at] ||
+          event_hash["occurred_at"] || event_hash[:occurred_at],
+        event_hash["metadata"] || event_hash[:metadata] || {}
+      )
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
     def validate_cdc_event!(event)
       return event if event.respond_to?(:to_h) && cdc_event_hash?(event.to_h)
 
@@ -74,5 +106,8 @@ module Mammoth
     def transaction_envelope?(work)
       work.respond_to?(:events) && work.respond_to?(:transaction_id)
     end
+
+    SyntheticTransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
+    private_constant :SyntheticTransactionEnvelope
   end
 end