mammoth 0.9.0 → 1.1.0

data/lib/mammoth/delivery_worker.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 module Mammoth
-  # Delivers normalized events with retry, checkpoint, and dead-letter handling.
+  # Delivers normalized events with retry, ledger, and dead-letter handling.
   #
   # DeliveryWorker is Mammoth's first reliable delivery unit. It intentionally keeps
-  # the delivery contract small: attempt webhook delivery, advance the checkpoint
-  # after success, and persist the failed event to the dead letter queue after
-  # retry exhaustion.
+  # the delivery contract small: attempt webhook delivery, record idempotent
+  # success, and persist exhausted failures to the dead letter queue. Contiguous
+  # checkpointing is owned by DeliveryProgressCoordinator.
   class DeliveryWorker
     # Default source name used when an event does not provide one.
     DEFAULT_SOURCE = "postgresql"
@@ -15,7 +15,7 @@ module Mammoth
                 :sleeper, :source_name, :slot_name, :publication_name, :route_filter, :enabled
 
     # @param sink [#deliver] destination sink
-    # @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
+    # @param checkpoint_store [Mammoth::CheckpointStore] retained dependency; shared progress owns writes
     # @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
     # @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore] downstream delivery ledger
     # @param source_name [String] logical source name
@@ -47,7 +47,7 @@ module Mammoth
     #
     # @param config [Mammoth::Configuration] loaded configuration
     # @param sink [#deliver] destination sink
-    # @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
+    # @param checkpoint_store [Mammoth::CheckpointStore] retained dependency; shared progress owns writes
     # @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
     # @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore] downstream delivery ledger
     # @param sleeper [#call] sleep strategy
@@ -70,7 +70,7 @@ module Mammoth
       )
     end
 
-    # Deliver a transaction envelope with retry, checkpoint, and DLQ handling.
+    # Deliver a transaction envelope with retry, ledger, and DLQ handling.
     #
     # @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
     # @return [Hash] delivery summary
@@ -78,7 +78,7 @@ module Mammoth
       deliver_work(envelope, serializer: TransactionEnvelopeSerializer, delivery_method: :deliver_transaction)
     end
 
-    # Deliver an event with retry, checkpoint, and DLQ handling.
+    # Deliver an event with retry, ledger, and DLQ handling.
     #
     # @param event [CDC::Core::ChangeEvent] normalized event
     # @return [Hash] delivery summary
@@ -99,7 +99,6 @@ module Mammoth
       return skip_result if skip_result
 
       if delivered_envelope_store.delivered?(idempotency_key)
-        checkpoint_payload(payload)
         return {
           status: "skipped",
           duplicate: true,
@@ -121,7 +120,6 @@ module Mammoth
           transaction_id: payload["transaction_id"],
           source_position: payload["source_position"]
         )
-        checkpoint_payload(payload)
         result.merge(attempts: attempts, idempotency_key: idempotency_key)
       rescue DeliveryError => e
         return dead_letter(work, e, attempts, serializer:) if attempts >= max_attempts
@@ -132,19 +130,6 @@ module Mammoth
     end
     # rubocop:enable Metrics/MethodLength
 
-    def checkpoint(work, serializer:)
-      checkpoint_payload(serializer.call(work))
-    end
-
-    def checkpoint_payload(payload)
-      checkpoint_store.write(
-        source_name: source_name,
-        slot_name: slot_name,
-        publication_name: publication_name,
-        last_lsn: payload["source_position"]
-      )
-    end
-
     def idempotency_key_for(payload:, delivery_unit:)
       [
         source_name,
@@ -170,8 +155,7 @@ module Mammoth
       skipped(payload, idempotency_key:, reason: "route_mismatch") unless route_filter.match_payload?(payload)
     end
 
-    def skipped(payload, idempotency_key:, reason:)
-      checkpoint_payload(payload)
+    def skipped(_payload, idempotency_key:, reason:)
       {
         status: "skipped",
         reason: reason,

data/lib/mammoth/event_serializer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "json"
-require "securerandom"
+require "digest"
 require "time"
 
 module Mammoth
@@ -27,17 +27,17 @@ module Mammoth
     def initialize(event)
       raise ArgumentError, "event must be a CDC::Core::ChangeEvent" unless event.is_a?(CDC::Core::ChangeEvent)
 
-      @event = event.to_h
+      @event = event
     end
 
     # Return the webhook payload.
     #
     # @return [Hash] webhook payload
     def call
-      event_hash = stringify_keys(@event)
+      event_hash = stringify_keys(@event.to_h)
       metadata = stringify_keys(event_hash["metadata"] || {})
       {
-        "event_id" => event_id(metadata),
+        "event_id" => event_id(event_hash, metadata),
         "source" => source(metadata),
         "operation" => normalize_operation(event_hash.fetch("operation")),
         "namespace" => event_hash["schema"],
@@ -47,7 +47,7 @@ module Mammoth
         "transaction_id" => event_hash["transaction_id"],
         "occurred_at" => occurred_at(event_hash),
         "data" => event_data(event_hash),
-        "changes" => metadata["changes"] || [],
+        "changes" => serialized_changes(event_hash, metadata),
         "metadata" => metadata
       }
     end
@@ -69,8 +69,25 @@ module Mammoth
       operation.to_s
     end
 
-    def event_id(metadata)
-      metadata["event_id"] || SecureRandom.uuid
+    def event_id(event_hash, metadata)
+      metadata["event_id"] || deterministic_event_id(event_hash, metadata)
+    end
+
+    def deterministic_event_id(event_hash, metadata)
+      identity = {
+        source: source(metadata),
+        transaction_id: event_hash["transaction_id"],
+        source_position: event_hash["commit_lsn"],
+        sequence_number: event_hash["sequence_number"],
+        occurred_at: event_hash["occurred_at"],
+        namespace: event_hash["schema"],
+        entity: event_hash["table"],
+        operation: normalize_operation(event_hash.fetch("operation")),
+        primary_key: event_hash["primary_key"],
+        old_values: event_hash["old_values"],
+        new_values: event_hash["new_values"]
+      }
+      "evt_#{Digest::SHA256.hexdigest(JSON.generate(identity))}"
     end
 
     def source(metadata)
@@ -87,5 +104,20 @@ module Mammoth
     def event_data(event_hash)
       event_hash["data"] || event_hash["new_values"] || event_hash["old_values"] || {}
     end
+
+    def serialized_changes(event_hash, metadata)
+      return metadata["changes"] || [] if metadata.key?("changes")
+      return [] unless changes_available?(event_hash)
+
+      @event.changes.map(&:to_h)
+    end
+
+    def changes_available?(event_hash)
+      return true unless normalize_operation(event_hash.fetch("operation")) == "update"
+
+      old_values = event_hash["old_values"]
+      new_values = event_hash["new_values"]
+      old_values.is_a?(Hash) && new_values.is_a?(Hash) && old_values.keys.sort == new_values.keys.sort
+    end
   end
 end

data/lib/mammoth/observability_metrics.rb

@@ -3,6 +3,8 @@
 module Mammoth
   # Prometheus formatting helpers for operational and dispatch metrics.
   module ObservabilityMetrics
+    include PostgresObservabilityMetrics
+
     # Maps canonical CDC core metric names to Mammoth's Prometheus counters.
     DISPATCH_METRIC_NAMES = {
       CDC::Core::Observer.started_metric_name => "mammoth_dispatch_started_total",
@@ -14,7 +16,7 @@ module Mammoth
     private
 
     def metric_headers
-      operational_metric_headers + dispatch_metric_headers
+      operational_metric_headers + dispatch_metric_headers + postgres_metric_headers
     end
 
     def operational_metric_headers

data/lib/mammoth/observability_server.rb

@@ -15,18 +15,20 @@ module Mammoth
     # Default TCP port for the observability server.
     DEFAULT_PORT = 9393
 
-    attr_reader :config, :host, :port, :state_adapter, :logger, :server
+    attr_reader :config, :host, :port, :state_adapter, :slot_health_provider, :logger, :server
 
     # @param config [Mammoth::Configuration] loaded configuration
     # @param host [String, nil] bind host override
     # @param port [Integer, nil] bind port override
     # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
+    # @param slot_health_provider [#slot_health, nil] PostgreSQL slot health dependency
     # @param logger [WEBrick::Log, nil] optional WEBrick logger
-    def initialize(config, host: nil, port: nil, state_adapter: nil, logger: nil)
+    def initialize(config, host: nil, port: nil, state_adapter: nil, slot_health_provider: nil, logger: nil)
       @config = config
       @host = host || config.dig("observability", "host") || DEFAULT_HOST
       @port = port || config.dig("observability", "port") || DEFAULT_PORT
       @state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
+      @slot_health_provider = slot_health_provider || Sources::Postgres.new(config)
       @logger = logger || WEBrick::Log.new($stderr, WEBrick::Log::WARN)
       @server = build_server
       mount_endpoints
@@ -81,7 +83,7 @@ module Mammoth
     end
 
     def snapshot
-      ObservabilitySnapshot.new(config, state_adapter: state_adapter)
+      ObservabilitySnapshot.new(config, state_adapter:, slot_health_provider:)
     end
   end
 end

data/lib/mammoth/observability_snapshot.rb

@@ -3,27 +3,30 @@
 require "time"
 
 module Mammoth
-  # Builds health, readiness, and metrics snapshots from Mammoth's local
-  # operational state.
+  # Builds health, readiness, and metrics snapshots from Mammoth's operational
+  # state and optional PostgreSQL slot-health provider.
   #
   # ObservabilitySnapshot is intentionally read-only. It does not start the
-  # relay, mutate checkpoints, replay dead letters, or inspect PostgreSQL. The
-  # health and metrics endpoints use this object to expose Mammoth process and
-  # operational-state adapter status in a predictable format.
-  class ObservabilitySnapshot
+  # relay, mutate checkpoints, replay dead letters, or change PostgreSQL slot
+  # state. An injected provider may perform read-only slot inspection. The
+  # endpoints expose process, operational-state, and source status predictably.
+  class ObservabilitySnapshot # rubocop:disable Metrics/ClassLength
     include ObservabilityMetrics
 
-    attr_reader :config, :state_adapter, :clock, :dispatch_metrics
+    attr_reader :config, :state_adapter, :clock, :dispatch_metrics, :slot_health_provider
 
     # @param config [Mammoth::Configuration] loaded configuration
     # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
     # @param clock [#call] time source returning a Time-like object
     # @param dispatch_metrics [Mammoth::DispatchMetrics] dispatch counter registry
-    def initialize(config, state_adapter: nil, clock: -> { Time.now.utc }, dispatch_metrics: DispatchMetrics::INSTANCE)
+    # @param slot_health_provider [#slot_health, nil] PostgreSQL slot health dependency
+    def initialize(config, state_adapter: nil, clock: -> { Time.now.utc }, dispatch_metrics: DispatchMetrics::INSTANCE,
+                   slot_health_provider: nil)
       @config = config
       @state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
       @clock = clock
       @dispatch_metrics = dispatch_metrics
+      @slot_health_provider = slot_health_provider
     end
 
     # Build a liveness response.
@@ -45,7 +48,7 @@ module Mammoth
     def readiness
       return unready_payload unless state_adapter.ready?
 
-      {
+      payload = {
         status: "ready",
         service: "mammoth",
         name: mammoth_name,
@@ -54,6 +57,14 @@ module Mammoth
         summary: state_summary,
         checked_at: checked_at
       }
+      return payload unless slot_health_provider
+
+      health = postgres_slot_health
+      return postgres_unready_payload(health) unless health.ready?
+
+      payload.merge(postgres_slot: health.summary)
+    rescue ReplicationError => e
+      postgres_error_payload(e)
     rescue Mammoth::Error => e
       adapter_error_payload(e)
     end
@@ -71,7 +82,7 @@ module Mammoth
       ) + destination_metric_lines(
         dead_letter_store: state_adapter.dead_letter_store,
         delivered_store: state_adapter.delivered_envelope_store
-      ) + dispatch_metric_lines
+      ) + dispatch_metric_lines + postgres_slot_metric_lines
       "#{lines.join("\n")}\n"
     rescue Mammoth::Error
       down_metrics
@@ -92,6 +103,36 @@ module Mammoth
       }
     end
 
+    def postgres_unready_payload(health)
+      {
+        status: "unready",
+        service: "mammoth",
+        name: mammoth_name,
+        operational_state: "ok",
+        adapter: state_summary.fetch(:adapter),
+        summary: state_summary,
+        postgres_slot: health.summary,
+        checked_at: checked_at
+      }
+    end
+
+    def postgres_error_payload(error)
+      {
+        status: "unready",
+        service: "mammoth",
+        name: mammoth_name,
+        operational_state: "ok",
+        adapter: configured_adapter_name,
+        postgres_slot: {
+          ready: false,
+          reason: "inspection failed",
+          error_class: error.class.name,
+          error_message: error.message
+        },
+        checked_at: checked_at
+      }
+    end
+
     def aggregate_metric_lines(checkpoint_store:, dead_letter_store:, delivered_store:)
       [
         metric_line("mammoth_up", 1),
@@ -136,6 +177,18 @@ module Mammoth
       "#{(metric_headers + [metric_line("mammoth_up", 0)] + dispatch_metric_lines).join("\n")}\n"
     end
 
+    def postgres_slot_metric_lines
+      return [] unless slot_health_provider
+
+      postgres_metric_lines(postgres_slot_health)
+    rescue ReplicationError
+      postgres_inspection_error_metric_lines
+    end
+
+    def postgres_slot_health
+      slot_health_provider.slot_health
+    end
+
     def state_summary
       @state_summary ||= state_adapter.summary
     end

data/lib/mammoth/postgres_observability_metrics.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # PostgreSQL slot-specific Prometheus formatting helpers.
+  module PostgresObservabilityMetrics
+    # Metric names and help text for PostgreSQL replication-slot gauges.
+    POSTGRES_METRICS = {
+      "mammoth_postgres_slot_inspection_up" => "1 when PostgreSQL slot inspection succeeds.",
+      "mammoth_postgres_slot_present" => "1 when the configured replication slot exists.",
+      "mammoth_postgres_slot_ready" => "1 when the configured slot is healthy and active.",
+      "mammoth_postgres_slot_active" => "1 when PostgreSQL reports the slot as active.",
+      "mammoth_postgres_slot_retained_wal_bytes" => "WAL retained from the slot restart LSN.",
+      "mammoth_postgres_slot_safe_wal_size_bytes" => "WAL bytes that may be written before the slot risks loss.",
+      "mammoth_postgres_slot_wal_status" => "Current PostgreSQL WAL retention status for the slot.",
+      "mammoth_postgres_slot_invalidated" => "1 when PostgreSQL reports invalidation or conflict.",
+      "mammoth_postgres_slot_inactive_since_timestamp_seconds" => "Unix timestamp when the slot became inactive.",
+      "mammoth_postgres_slot_restart_lsn_bytes" => "Numeric PostgreSQL restart LSN.",
+      "mammoth_postgres_slot_confirmed_flush_lsn_bytes" => "Numeric PostgreSQL confirmed flush LSN."
+    }.freeze
+
+    private
+
+    def postgres_metric_headers
+      POSTGRES_METRICS.flat_map { |name, help| ["# HELP #{name} #{help}", "# TYPE #{name} gauge"] }
+    end
+
+    def postgres_metric_lines(health)
+      labels = { slot_name: health.slot_name }
+      lines = postgres_presence_metric_lines(health, labels:)
+      return lines unless health.present
+
+      lines + postgres_status_metric_lines(health, labels:) + postgres_optional_metric_lines(health, labels:)
+    end
+
+    def postgres_presence_metric_lines(health, labels:)
+      [
+        metric_line("mammoth_postgres_slot_inspection_up", 1, labels:),
+        metric_line("mammoth_postgres_slot_present", health.present ? 1 : 0, labels:)
+      ]
+    end
+
+    def postgres_status_metric_lines(health, labels:)
+      [
+        metric_line("mammoth_postgres_slot_ready", health.ready? ? 1 : 0, labels:),
+        metric_line("mammoth_postgres_slot_active", health.active ? 1 : 0, labels:),
+        metric_line("mammoth_postgres_slot_wal_status", 1,
+                    labels: labels.merge(wal_status: health.wal_status || "unknown")),
+        metric_line("mammoth_postgres_slot_invalidated", invalidated?(health) ? 1 : 0, labels:)
+      ]
+    end
+
+    def postgres_optional_metric_lines(health, labels:)
+      {
+        "mammoth_postgres_slot_retained_wal_bytes" => health.retained_wal_bytes,
+        "mammoth_postgres_slot_safe_wal_size_bytes" => health.safe_wal_size,
+        "mammoth_postgres_slot_inactive_since_timestamp_seconds" => timestamp_seconds(health.inactive_since),
+        "mammoth_postgres_slot_restart_lsn_bytes" => health.restart_lsn_bytes,
+        "mammoth_postgres_slot_confirmed_flush_lsn_bytes" => health.confirmed_flush_lsn_bytes
+      }.filter_map { |name, value| metric_line(name, value, labels:) unless value.nil? }
+    end
+
+    def postgres_inspection_error_metric_lines
+      labels = { slot_name: config.dig("replication", "slot") }
+      [metric_line("mammoth_postgres_slot_inspection_up", 0, labels:)]
+    end
+
+    def timestamp_seconds(value)
+      return nil if value.nil?
+      return value.to_i if value.respond_to?(:to_i) && !value.is_a?(String)
+
+      Time.parse(value.to_s).to_i
+    rescue ArgumentError
+      nil
+    end
+
+    def invalidated?(health)
+      health.conflicting || (!health.invalidation_reason.nil? && health.invalidation_reason != "")
+    end
+  end
+end

data/lib/mammoth/replication_consumer.rb

@@ -27,7 +27,7 @@ module Mammoth
 
       count = 0
 
-      each_event do |event|
+      each_grouped_event do |event, _group_end|
         yield event
         count += 1
       end
@@ -35,11 +35,27 @@ module Mammoth
       count
     end
 
+    # Consume work with an explicit source-group boundary for safe progress.
+    #
+    # @yieldparam event [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
+    # @yieldparam group_end [Boolean] true for the final delivery in a source transaction
+    # @return [Integer] number of consumed work items
+    def start_with_boundaries
+      return enum_for(:start_with_boundaries) unless block_given?
+
+      count = 0
+      each_grouped_event do |event, group_end|
+        yield event, group_end
+        count += 1
+      end
+      count
+    end
+
     private
 
-    def each_event(&block)
+    def each_grouped_event(&block)
       effective_source.each do |work|
-        flatten_cdc_work(work).each(&block)
+        grouped_cdc_work(work).each { |event, group_end| block.call(event, group_end) }
       end
     end
 
@@ -47,15 +63,20 @@ module Mammoth
       source || raise(ReplicationError, "replication source is not configured")
     end
 
-    def flatten_cdc_work(work)
+    def grouped_cdc_work(work)
       return [] if work.nil?
-      return work.flat_map { |item| flatten_cdc_work(item) } if work.is_a?(Array)
-      return transaction_work(work) if work.is_a?(CDC::Core::TransactionEnvelope)
-      return event_work(work) if work.is_a?(CDC::Core::ChangeEvent)
+      return work.flat_map { |item| grouped_cdc_work(item) } if work.is_a?(Array)
+      return grouped_transaction_work(work) if work.is_a?(CDC::Core::TransactionEnvelope)
+      return event_work(work).map { |item| [item, true] } if work.is_a?(CDC::Core::ChangeEvent)
 
       raise ReplicationError, "CDC source yielded non-core work: #{work.class}"
     end
 
+    def grouped_transaction_work(envelope)
+      items = transaction_work(envelope)
+      items.each_with_index.map { |item, index| [item, index == items.length - 1] }
+    end
+
     def transaction_delivery?
       delivery_unit == :transaction
     end