mammoth 0.8.0 → 0.9.0

data/lib/mammoth/persisted_payload_deserializer.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Mammoth
+  # Reconstructs exact CDC core work items from persisted webhook payloads.
+  #
+  # Dead letters and sample files contain Mammoth's JSON delivery projection,
+  # not live CDC objects. This class is the explicit boundary that converts
+  # those persisted representations back into core vocabulary.
+  class PersistedPayloadDeserializer
+    class << self
+      # Deserialize one persisted event payload.
+      #
+      # @param payload [Hash] Mammoth event payload
+      # @return [CDC::Core::ChangeEvent] reconstructed core event
+      def event(payload)
+        attributes = stringify_keys(payload)
+        data = attributes["data"]
+        metadata = enriched_metadata(attributes)
+
+        CDC::Core::ChangeEvent.new(
+          operation: attributes.fetch("operation"),
+          schema: attributes["namespace"] || attributes.fetch("schema"),
+          table: attributes["entity"] || attributes.fetch("table"),
+          old_values: attributes["old_values"] || delete_values(attributes, data),
+          new_values: attributes["new_values"] || changed_values(attributes, data),
+          primary_key: attributes["identity"] || attributes["primary_key"],
+          transaction_id: attributes["transaction_id"],
+          commit_lsn: attributes["commit_lsn"] || attributes["source_position"],
+          sequence_number: attributes["sequence_number"],
+          occurred_at: parse_time(attributes["occurred_at"]),
+          metadata: metadata
+        )
+      rescue KeyError, ArgumentError, TypeError => e
+        raise ConfigurationError, "invalid persisted CDC event: #{e.message}"
+      end
+
+      # Deserialize one persisted transaction payload.
+      #
+      # @param payload [Hash] Mammoth transaction webhook payload
+      # @return [CDC::Core::TransactionEnvelope] reconstructed core transaction
+      def transaction(payload)
+        attributes = stringify_keys(payload)
+        CDC::Core::TransactionEnvelope.new(
+          transaction_id: attributes.fetch("transaction_id"),
+          events: attributes.fetch("events").map { |event_payload| event(event_payload) },
+          commit_lsn: attributes["commit_lsn"] || attributes["source_position"],
+          committed_at: parse_time(attributes["committed_at"]),
+          metadata: enriched_metadata(attributes)
+        )
+      rescue KeyError, ArgumentError, TypeError => e
+        raise ConfigurationError, "invalid persisted CDC transaction: #{e.message}"
+      end
+
+      private
+
+      def stringify_keys(payload)
+        payload.to_h.transform_keys(&:to_s)
+      end
+
+      def enriched_metadata(attributes)
+        metadata = stringify_keys(attributes["metadata"] || {})
+        %w[event_id source source_position changes type].each do |key|
+          metadata[key] = attributes[key] if attributes.key?(key)
+        end
+        metadata
+      end
+
+      def delete_values(attributes, data)
+        data if attributes["operation"].to_s == "delete"
+      end
+
+      def changed_values(attributes, data)
+        data unless attributes["operation"].to_s == "delete"
+      end
+
+      def parse_time(value)
+        return value if value.nil? || value.is_a?(Time)
+
+        Time.iso8601(value.to_s)
+      end
+    end
+  end
+end

data/lib/mammoth/replication_consumer.rb

@@ -20,7 +20,7 @@ module Mammoth
 
     # Consume normalized CDC work from the configured source.
     #
-    # @yieldparam event [Object] CDC::Core::ChangeEvent-compatible event
+    # @yieldparam event [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
     # @return [Integer] number of consumed events
     def start
       return enum_for(:start) unless block_given?
@@ -49,65 +49,33 @@ module Mammoth
 
     def flatten_cdc_work(work)
       return [] if work.nil?
-      return validate_transaction_envelope(work) if transaction_envelope?(work) && transaction_delivery?
-      return validate_events(work.events) if transaction_envelope?(work)
       return work.flat_map { |item| flatten_cdc_work(item) } if work.is_a?(Array)
-      return [synthetic_transaction_envelope(work)] if transaction_delivery?
+      return transaction_work(work) if work.is_a?(CDC::Core::TransactionEnvelope)
+      return event_work(work) if work.is_a?(CDC::Core::ChangeEvent)
 
-      validate_events([work])
+      raise ReplicationError, "CDC source yielded non-core work: #{work.class}"
     end
 
     def transaction_delivery?
       delivery_unit == :transaction
     end
 
-    def validate_transaction_envelope(envelope)
-      validate_events(envelope.events)
-      [envelope]
+    def transaction_work(envelope)
+      transaction_delivery? ? [envelope] : envelope.events
     end
 
-    def validate_events(events)
-      events.each { |event| validate_cdc_event!(event) }
+    def event_work(event)
+      transaction_delivery? ? [transaction_envelope(event)] : [event]
     end
 
-    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-    def synthetic_transaction_envelope(event)
-      validate_cdc_event!(event)
-
-      event_hash = event.to_h
-      SyntheticTransactionEnvelope.new(
-        [event],
-        event_hash["transaction_id"] || event_hash[:transaction_id] ||
-          event_hash["xid"] || event_hash[:xid] || event_hash["event_id"] || event_hash[:event_id],
-        event_hash["commit_lsn"] || event_hash[:commit_lsn] ||
-          event_hash["source_position"] || event_hash[:source_position],
-        event_hash["committed_at"] || event_hash[:committed_at] ||
-          event_hash["occurred_at"] || event_hash[:occurred_at],
-        event_hash["metadata"] || event_hash[:metadata] || {}
+    def transaction_envelope(event)
+      CDC::Core::TransactionEnvelope.new(
+        transaction_id: event.transaction_id || event.metadata[:event_id] || event.commit_lsn,
+        events: [event],
+        commit_lsn: event.commit_lsn,
+        committed_at: event.occurred_at,
+        metadata: event.metadata
       )
     end
-    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
-    def validate_cdc_event!(event)
-      return event if event.respond_to?(:to_h) && cdc_event_hash?(event.to_h)
-
-      raise ReplicationError, "CDC source yielded non-CDC work: #{event.class}"
-    end
-
-    def cdc_event_hash?(event_hash)
-      return false unless event_hash.respond_to?(:key?)
-
-      has_operation = event_hash.key?("operation") || event_hash.key?(:operation)
-      has_position = event_hash.key?("source_position") || event_hash.key?(:source_position) ||
-                     event_hash.key?("commit_lsn") || event_hash.key?(:commit_lsn)
-      has_operation && has_position
-    end
-
-    def transaction_envelope?(work)
-      work.respond_to?(:events) && work.respond_to?(:transaction_id)
-    end
-
-    SyntheticTransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
-    private_constant :SyntheticTransactionEnvelope
   end
 end

data/lib/mammoth/runtimes/batching_runtime.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Mammoth
+  module Runtimes
+    # Adds configured batch submission to a selected delivery runtime.
+    #
+    # Application streams one work item at a time into this runtime boundary.
+    # BatchingRuntime owns accumulation and submits complete or final partial
+    # batches through the selected adapter runtime's #process_many contract.
+    class BatchingRuntime
+      attr_reader :runtime, :batch_size, :buffer
+
+      # @param runtime [#process_many] selected delivery runtime
+      # @param batch_size [Integer] maximum work items submitted together
+      def initialize(runtime:, batch_size:)
+        @runtime = runtime
+        @batch_size = batch_size
+        @buffer = []
+      end
+
+      # Buffer one work item and submit a full batch when ready.
+      #
+      # @param work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
+      # @return [Array] processor results when a batch is submitted, otherwise an empty array
+      def process(work)
+        buffer << work
+        return [] if buffer.size < batch_size
+
+        flush
+      end
+
+      # Submit work immediately without changing the buffered stream.
+      #
+      # @param items [Array] work items
+      # @return [Array] processor results
+      def process_many(items)
+        runtime.process_many(items)
+      end
+
+      # Submit the final partial batch.
+      #
+      # @return [Array] processor results, or an empty array when no work is buffered
+      def flush
+        return [] if buffer.empty?
+
+        runtime.process_many(buffer.shift(buffer.size))
+      end
+
+      # Flush pending work and shut down the selected runtime when supported.
+      #
+      # @return [nil]
+      def shutdown
+        flush
+        runtime.shutdown if runtime.respond_to?(:shutdown)
+        nil
+      end
+    end
+  end
+end

data/lib/mammoth/runtimes/concurrent_adapter.rb

@@ -16,13 +16,15 @@ module Mammoth
         # @param concurrency [Integer] worker count
         # @param timeout [Numeric, nil] optional timeout
         # @param preserve_order [Boolean] preserve ordering when supported
+        # @param observer [CDC::Core::Observer] dispatch lifecycle observer
         # @return [Mammoth::ConcurrentDeliveryRuntime]
-        def build(processor:, concurrency:, timeout:, preserve_order:)
+        def build(processor:, concurrency:, timeout:, preserve_order:, observer: CDC::Core::Observer.new)
           ConcurrentDeliveryRuntime.new(
             processor: processor,
             concurrency: concurrency,
             timeout: timeout,
-            preserve_order: preserve_order
+            preserve_order: preserve_order,
+            observer: observer
           )
         end
       end

data/lib/mammoth/runtimes/inline_adapter.rb

@@ -4,20 +4,22 @@ module Mammoth
   module Runtimes
     # Inline runtime adapter that processes work in the caller thread.
     class InlineAdapter < Adapter
-      attr_reader :processor
+      attr_reader :processor, :observer
 
       # @param processor [#process] delivery processor
-      def initialize(processor:)
+      # @param observer [CDC::Core::Observer] dispatch lifecycle observer
+      def initialize(processor:, observer: CDC::Core::Observer.new)
         super()
         @processor = processor
+        @observer = observer
       end
 
       # Build an inline runtime.
       #
       # @param processor [#process] delivery processor
       # @return [Mammoth::Runtimes::InlineAdapter]
-      def self.build(processor:, **_options)
-        new(processor: processor)
+      def self.build(processor:, observer: CDC::Core::Observer.new, **_options)
+        new(processor: processor, observer: observer)
       end
 
       # @return [String] adapter type name
@@ -28,13 +30,25 @@ module Mammoth
       # @param items [Array<Object>] work units
       # @return [Array<Object>] processor results
       def process_many(items)
-        items.map { |item| processor.process(item) }
+        items.map do |item|
+          observer.dispatch_started(item)
+          processor.process(item).tap { |result| observe_result(result) }
+        end
       end
 
       # @return [nil]
       def shutdown
         nil
       end
+
+      private
+
+      def observe_result(result)
+        return observer.dispatch_succeeded(result) if result.success?
+        return observer.dispatch_failed(result) if result.failure?
+
+        observer.dispatch_skipped(result)
+      end
     end
   end
 end

data/lib/mammoth/runtimes/registry.rb

@@ -27,8 +27,9 @@ module Mammoth
         # @param name [String, Symbol] adapter name
         # @param options [Hash] adapter-specific build options
         # @return [Object] runtime adapter
-        def build(name, **options)
-          fetch(name).build(**options)
+        def build(name, batch_size: 1, **options)
+          runtime = fetch(name).build(**options)
+          BatchingRuntime.new(runtime:, batch_size:)
         end
 
         # @return [Array<String>] registered runtime adapter names

data/lib/mammoth/sources/postgres.rb

@@ -64,7 +64,9 @@ module Mammoth
           raise ReplicationError, "pgoutput source adapter must respond to #each_normalized"
         end
 
-        normalizer.each_normalized(decoded_stream, &block)
+        normalizer.each_normalized(decoded_stream) do |work|
+          block.call(validate_core_work!(work))
+        end
         nil
       rescue StandardError => e
         raise e if e.is_a?(ReplicationError)
@@ -106,6 +108,13 @@ module Mammoth
         normalizer.stream_event(decoded, source_position: source_position(metadata))
       end
 
+      def validate_core_work!(work)
+        return work if work.is_a?(CDC::Core::ChangeEvent)
+        return work if work.is_a?(CDC::Core::TransactionEnvelope)
+
+        raise ReplicationError, "pgoutput source adapter yielded non-core work: #{work.class}"
+      end
+
       def parse_payload(payload)
         parser = effective_parser
         return parser.process(payload) if parser.respond_to?(:process)

data/lib/mammoth/status.rb

@@ -3,23 +3,23 @@
 module Mammoth
   # Builds and prints a boring operational status snapshot.
   class Status
-    attr_reader :config, :sqlite_store, :output
+    attr_reader :config, :state_adapter, :output
 
-    # Print status for a configuration and optional SQLite store.
+    # Print status for a configuration and optional operational-state adapter.
     #
     # @param config [Mammoth::Configuration] loaded configuration
-    # @param sqlite_store [Mammoth::SQLiteStore, nil] operational store
+    # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
     # @return [void]
-    def self.call(config, sqlite_store: nil, output: $stdout)
-      new(config, sqlite_store: sqlite_store, output: output).call
+    def self.call(config, state_adapter: nil, output: $stdout)
+      new(config, state_adapter: state_adapter, output: output).call
     end
 
     # @param config [Mammoth::Configuration] loaded configuration
-    # @param sqlite_store [Mammoth::SQLiteStore, nil] operational store
+    # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
     # @param output [#puts] output stream
-    def initialize(config, sqlite_store: nil, output: $stdout)
+    def initialize(config, state_adapter: nil, output: $stdout)
       @config = config
-      @sqlite_store = sqlite_store
+      @state_adapter = state_adapter
       @output = output
     end
 
@@ -28,13 +28,13 @@ module Mammoth
     # @return [void]
     def call
       status_lines.each { |line| output.puts(line) }
-      print_store_state if sqlite_store
+      print_store_state if state_adapter
     end
 
     private
 
     def status_lines
-      identity_lines + replication_lines + runtime_lines + destination_lines + ["SQLite: #{sqlite_path}"]
+      identity_lines + replication_lines + runtime_lines + destination_lines
     end
 
     def identity_lines
@@ -70,10 +70,6 @@ module Mammoth
       ]
     end
 
-    def sqlite_path
-      config.dig("sqlite", "path")
-    end
-
     def destination_names
       destinations = config.data["destinations"]
       return destinations.map { |destination| destination.fetch("name") } if destinations
@@ -90,10 +86,18 @@ module Mammoth
     end
 
     def print_store_state
-      store = sqlite_store.bootstrap!
-      output.puts "Tables: #{store.tables.join(", ")}"
-      output.puts "Checkpoints: #{CheckpointStore.new(store).count}"
-      output.puts "Dead Letters: #{DeadLetterStore.new(store).count}"
+      output.puts "Operational state ready: #{state_adapter.ready?}"
+      state_adapter.summary.each do |key, value|
+        output.puts "#{status_label(key)}: #{format_status_value(value)}"
+      end
+    end
+
+    def status_label(key)
+      key.to_s.split("_").map(&:capitalize).join(" ")
+    end
+
+    def format_status_value(value)
+      value.is_a?(Array) ? value.join(", ") : value
     end
   end
 end

data/lib/mammoth/transaction_envelope_serializer.rb

@@ -15,16 +15,20 @@ module Mammoth
     # Default payload type for transaction webhook delivery.
     PAYLOAD_TYPE = "transaction.committed"
 
-    # Serialize a CDC::Core::TransactionEnvelope-like object into a Hash.
+    # Serialize a CDC::Core::TransactionEnvelope into a Hash.
     #
-    # @param envelope [#events, #transaction_id] transaction envelope
+    # @param envelope [CDC::Core::TransactionEnvelope] transaction envelope
     # @return [Hash] webhook-ready transaction payload
     def self.call(envelope)
       new(envelope).call
     end
 
-    # @param envelope [#events, #transaction_id] transaction envelope
+    # @param envelope [CDC::Core::TransactionEnvelope] transaction envelope
     def initialize(envelope)
+      unless envelope.is_a?(CDC::Core::TransactionEnvelope)
+        raise ArgumentError, "envelope must be a CDC::Core::TransactionEnvelope"
+      end
+
       @envelope = envelope
     end
 
@@ -34,7 +38,7 @@ module Mammoth
     def call
       event_payloads = envelope.events.map { |event| EventSerializer.call(event) }
       {
-        "event_id" => envelope_value("event_id") || SecureRandom.uuid,
+        "event_id" => envelope_metadata["event_id"] || SecureRandom.uuid,
         "type" => PAYLOAD_TYPE,
         "source" => first_event_value(event_payloads, "source") || EventSerializer::DEFAULT_SOURCE,
         "transaction_id" => envelope.transaction_id,
@@ -43,7 +47,7 @@ module Mammoth
         "committed_at" => committed_at,
         "event_count" => event_payloads.length,
         "events" => event_payloads,
-        "metadata" => envelope_value("metadata") || {}
+        "metadata" => envelope_metadata
       }
     end
 
@@ -58,27 +62,21 @@ module Mammoth
 
     attr_reader :envelope
 
-    def envelope_hash
-      @envelope_hash ||= envelope.respond_to?(:to_h) ? stringify_keys(envelope.to_h) : {}
-    end
-
     def stringify_keys(hash)
       hash.to_h.transform_keys(&:to_s)
     end
 
-    def envelope_value(key)
-      return envelope.public_send(key) if envelope.respond_to?(key)
-
-      envelope_hash[key]
+    def envelope_metadata
+      @envelope_metadata ||= stringify_keys(envelope.metadata)
     end
 
     def source_position(event_payloads)
-      envelope_value("source_position") || envelope_value("commit_lsn") ||
+      envelope.commit_lsn ||
         first_event_value(event_payloads.reverse, "source_position")
     end
 
     def committed_at
-      value = envelope_value("committed_at") || envelope_value("commit_time")
+      value = envelope.committed_at
       return value.iso8601 if value.respond_to?(:iso8601)
 
       value || Time.now.utc.iso8601

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

data/lib/mammoth/webhook_sink.rb

@@ -94,7 +94,7 @@ module Mammoth
 
     # Deliver an event to the webhook endpoint.
     #
-    # @param event [Hash, #to_h] normalized event
+    # @param event [CDC::Core::ChangeEvent] normalized event
     # @return [Hash] delivery result
     # @raise [Mammoth::DeliveryError] when delivery fails
     def deliver(event)
@@ -103,7 +103,7 @@ module Mammoth
 
     # Deliver a transaction envelope to the webhook endpoint.
     #
-    # @param envelope [#events, #transaction_id] CDC transaction envelope
+    # @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
     # @return [Hash] delivery result
     # @raise [Mammoth::DeliveryError] when delivery fails
     def deliver_transaction(envelope)

data/lib/mammoth.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "cdc/core"
+
 require_relative "mammoth/version"
 require_relative "mammoth/errors"
 require_relative "mammoth/registry"
@@ -14,6 +16,9 @@ require_relative "mammoth/commands/status_command"
 require_relative "mammoth/commands/start_command"
 require_relative "mammoth/commands/deliver_sample_command"
 require_relative "mammoth/commands/dead_letters_command"
+require_relative "mammoth/dispatch_metrics"
+require_relative "mammoth/metrics_observer"
+require_relative "mammoth/observability_metrics"
 require_relative "mammoth/observability_snapshot"
 require_relative "mammoth/observability_server"
 require_relative "mammoth/sqlite_store"
@@ -22,6 +27,7 @@ require_relative "mammoth/dead_letter_store"
 require_relative "mammoth/delivered_envelope_store"
 require_relative "mammoth/event_serializer"
 require_relative "mammoth/transaction_envelope_serializer"
+require_relative "mammoth/persisted_payload_deserializer"
 require_relative "mammoth/route_filter"
 require_relative "mammoth/webhook_sink"
 require_relative "mammoth/operational_state/adapter"
@@ -37,6 +43,7 @@ require_relative "mammoth/concurrent_delivery_runtime"
 require_relative "mammoth/runtimes/adapter"
 require_relative "mammoth/runtimes/inline_adapter"
 require_relative "mammoth/runtimes/concurrent_adapter"
+require_relative "mammoth/runtimes/batching_runtime"
 require_relative "mammoth/runtimes/registry"
 require_relative "mammoth/sources/postgres"
 require_relative "mammoth/cdc_source"
@@ -47,7 +54,7 @@ require_relative "mammoth/cli"
 
 # Mammoth is a self-hosted PostgreSQL event relay.
 #
-# Mammoth 0.7.x is a single-node PostgreSQL CDC relay with webhook fanout,
+# Mammoth 0.8.x is a single-node PostgreSQL CDC relay with webhook fanout,
 # replayable operational SQLite state, health/metrics endpoints, and explicit
 # extension contracts for state, destination, runtime, lifecycle, configuration,
 # and local command integrations.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mammoth
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -185,20 +185,25 @@ files:
 - lib/mammoth/destinations/adapter.rb
 - lib/mammoth/destinations/registry.rb
 - lib/mammoth/destinations/webhook_adapter.rb
+- lib/mammoth/dispatch_metrics.rb
 - lib/mammoth/errors.rb
 - lib/mammoth/event_serializer.rb
 - lib/mammoth/fanout_delivery_worker.rb
 - lib/mammoth/lifecycle_hooks.rb
+- lib/mammoth/metrics_observer.rb
 - lib/mammoth/node_identity.rb
+- lib/mammoth/observability_metrics.rb
 - lib/mammoth/observability_server.rb
 - lib/mammoth/observability_snapshot.rb
 - lib/mammoth/operational_state/adapter.rb
 - lib/mammoth/operational_state/registry.rb
 - lib/mammoth/operational_state/sqlite_adapter.rb
+- lib/mammoth/persisted_payload_deserializer.rb
 - lib/mammoth/registry.rb
 - lib/mammoth/replication_consumer.rb
 - lib/mammoth/route_filter.rb
 - lib/mammoth/runtimes/adapter.rb
+- lib/mammoth/runtimes/batching_runtime.rb
 - lib/mammoth/runtimes/concurrent_adapter.rb
 - lib/mammoth/runtimes/inline_adapter.rb
 - lib/mammoth/runtimes/registry.rb