mammoth 0.1.1 → 0.3.0

data/lib/mammoth/sources/postgres.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Mammoth
+  # Namespace for Mammoth source adapters.
   module Sources
     # Concrete PostgreSQL CDC source for Mammoth.
     #
@@ -12,6 +13,7 @@ module Mammoth
     # This class may mention pgoutput implementation details because it is the
     # concrete PostgreSQL source adapter used by Mammoth. The rest of Mammoth
     # should remain source-agnostic and consume only the work yielded here.
+    # rubocop:disable Metrics/ClassLength
     class Postgres
       # @return [Mammoth::Configuration] loaded Mammoth configuration
       attr_reader :config
@@ -23,6 +25,8 @@ module Mammoth
       attr_reader :decoder
       # @return [Object, nil] injected CDC source adapter
       attr_reader :adapter
+      # @return [Mammoth::CheckpointStore, nil] checkpoint store used for restart resume
+      attr_reader :checkpoint_store
 
       # Build a PostgreSQL CDC source.
       #
@@ -31,12 +35,14 @@ module Mammoth
       # @param parser [Object, nil] injected pgoutput parser or relation tracker
       # @param decoder [Object, nil] injected pgoutput decoder
       # @param adapter [Object, nil] injected source adapter
-      def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil)
+      # @param checkpoint_store [Mammoth::CheckpointStore, nil] persisted checkpoints for restart resume
+      def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil, checkpoint_store: nil)
         @config = config
         @runner = runner
         @parser = parser
         @decoder = decoder
         @adapter = adapter
+        @checkpoint_store = checkpoint_store
       end
 
       # Stream CDC::Core-shaped work from PostgreSQL logical replication.
@@ -67,9 +73,39 @@ module Mammoth
       def process_payload(payload, metadata, &block)
         parsed = parse_payload(payload)
         decoded = decode_message(parsed, metadata)
-        normalize_decoded(decoded).each { |work| block.call(work) if work }
+        process_decoded(decoded, metadata, &block)
       end
 
+      # rubocop:disable Metrics/MethodLength
+      def process_decoded(decoded, metadata, &block)
+        return if decoded.nil?
+
+        if decoded.is_a?(Array)
+          decoded.each { |item| process_decoded(item, metadata, &block) }
+          return
+        end
+
+        if begin_message?(decoded)
+          start_transaction_buffer(decoded)
+          return
+        end
+
+        if commit_message?(decoded)
+          emit_transaction_buffer(decoded, metadata, &block)
+          return
+        end
+
+        normalize_decoded(decoded).each do |work|
+          work = enrich_work_position(work, metadata, decoded)
+          if transaction_buffer_active?
+            Array(@transaction_events) << work
+          else
+            block.call(work)
+          end
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+
       def parse_payload(payload)
         parser = effective_parser
         return parser.process(payload) if parser.respond_to?(:process)
@@ -107,20 +143,129 @@ module Mammoth
         result.is_a?(Array) ? result.compact : [result].compact
       end
 
+      def begin_message?(decoded)
+        message_kind(decoded).include?("begin")
+      end
+
+      def commit_message?(decoded)
+        message_kind(decoded).include?("commit")
+      end
+
+      def message_kind(decoded)
+        (value_from(decoded, :message_type, :type, :kind) || decoded.class.name.to_s.split("::").last).to_s.downcase
+      end
+
+      def start_transaction_buffer(decoded)
+        @transaction_events = []
+        @transaction_id = value_from(decoded, :transaction_id, :xid, :final_lsn)
+        @transaction_metadata = value_hash(decoded, :metadata) || {}
+      end
+
+      def emit_transaction_buffer(decoded, metadata, &block)
+        return unless transaction_buffer_active?
+
+        block.call(
+          TransactionEnvelope.new(
+            @transaction_events,
+            transaction_id_for(decoded),
+            commit_lsn_for(decoded, metadata),
+            value_from(decoded, :commit_time, :committed_at, :timestamp),
+            @transaction_metadata
+          )
+        )
+      ensure
+        clear_transaction_buffer
+      end
+
+      def transaction_id_for(decoded)
+        value_from(decoded, :transaction_id, :xid) || @transaction_id || first_event_value(:transaction_id, :xid)
+      end
+
+      def commit_lsn_for(decoded, metadata)
+        value_from(decoded, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
+          value_from(metadata, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
+          first_event_value(:commit_lsn, :source_position, :lsn)
+      end
+
+      def transaction_buffer_active?
+        !@transaction_events.nil?
+      end
+
+      def clear_transaction_buffer
+        @transaction_events = nil
+        @transaction_id = nil
+        @transaction_metadata = nil
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def enrich_work_position(work, metadata, decoded)
+        position = value_from(work, :source_position, :commit_lsn) ||
+                   value_from(metadata, :source_position, :commit_lsn, :lsn) ||
+                   value_from(decoded, :source_position, :commit_lsn, :lsn)
+        return work unless position
+
+        work_hash = work.respond_to?(:to_h) ? work.to_h : work
+        return work unless work_hash.is_a?(Hash)
+
+        key_style = work_hash.key?("operation") ? :string : :symbol
+        source_position_key = key_style == :string ? "source_position" : :source_position
+        commit_lsn_key = key_style == :string ? "commit_lsn" : :commit_lsn
+        return work if work_hash[source_position_key] || work_hash[commit_lsn_key]
+
+        work_hash.merge(source_position_key => position, commit_lsn_key => position)
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+      def first_event_value(*keys)
+        event = Array(@transaction_events).find do |event|
+          keys.any? { |key| value_from(event, key) }
+        end
+        value_from(event, *keys) if event
+      end
+
+      def value_from(object, *keys)
+        return nil if object.nil?
+
+        keys.each do |key|
+          return object.public_send(key) if object.respond_to?(key)
+
+          hash = object.respond_to?(:to_h) ? object.to_h : object
+          next unless hash.respond_to?(:key?)
+
+          return hash[key] if hash.key?(key)
+          return hash[key.to_s] if hash.key?(key.to_s)
+        end
+
+        nil
+      end
+
+      def value_hash(object, key)
+        value = value_from(object, key)
+        value.is_a?(Hash) ? value : nil
+      end
+
       def effective_runner
-        runner || (@effective_runner ||= build_runner)
+        runner || @effective_runner || begin
+          @effective_runner = build_runner
+        end
       end
 
       def effective_parser
-        parser || (@effective_parser ||= build_parser)
+        parser || @effective_parser || begin
+          @effective_parser = build_parser
+        end
       end
 
       def effective_decoder
-        decoder || (@effective_decoder ||= build_decoder)
+        decoder || @effective_decoder || begin
+          @effective_decoder = build_decoder
+        end
       end
 
       def effective_adapter
-        adapter || (@effective_adapter ||= build_adapter)
+        adapter || @effective_adapter || begin
+          @effective_adapter = build_adapter
+        end
       end
 
       def build_runner
@@ -152,15 +297,47 @@ module Mammoth
           database_url: database_url,
           slot_name: required_config("replication", "slot"),
           publication_names: required_publications,
-          start_lsn: config.dig("replication", "start_lsn"),
-          auto_create_slot: !config.dig("replication", "auto_create_slot").nil?,
-          temporary_slot: !config.dig("replication", "temporary_slot").nil?
+          start_lsn: replication_start_lsn,
+          auto_create_slot: config.dig("replication", "auto_create_slot") == true,
+          temporary_slot: config.dig("replication", "temporary_slot") == true
         }.tap do |options|
           feedback_interval = config.dig("replication", "feedback_interval")
           options[:feedback_interval] = feedback_interval unless feedback_interval.nil?
         end
       end
 
+      def replication_start_lsn
+        configured = config.dig("replication", "start_lsn")
+        return configured unless blank?(configured)
+
+        checkpoint_lsn
+      end
+
+      def checkpoint_lsn
+        return nil unless checkpoint_store
+
+        row = checkpoint_store.fetch(source_name: source_name, slot_name: required_config("replication", "slot"))
+        normalize_lsn(row&.fetch("last_lsn", nil))
+      end
+
+      def source_name
+        required_config("mammoth", "name")
+      end
+
+      def normalize_lsn(value)
+        return nil if blank?(value)
+
+        lsn = value.to_s
+        return lsn if lsn.include?("/")
+        return "0/#{lsn.to_i.to_s(16).upcase}" if lsn.match?(/\A\d+\z/)
+
+        lsn
+      end
+
+      def blank?(value)
+        value.nil? || value == ""
+      end
+
       def required_publications
         publications = required_config("replication", "publications")
         unless publications.is_a?(Array) && publications.any? &&
@@ -198,6 +375,9 @@ module Mammoth
       rescue LoadError => e
         raise ReplicationError, "#{gem_name} is required for PostgreSQL CDC source integration: #{e.message}"
       end
+      TransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
+      private_constant :TransactionEnvelope
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/mammoth/sql/__bootstrap__.sql

@@ -53,3 +53,27 @@ ON dead_letters(namespace, entity);
 
 CREATE INDEX IF NOT EXISTS idx_dead_letters_failed_at
 ON dead_letters(failed_at);
+
+
+CREATE TABLE IF NOT EXISTS delivered_envelopes (
+  id INTEGER PRIMARY KEY,
+  idempotency_key TEXT NOT NULL,
+  source_name TEXT NOT NULL,
+  slot_name TEXT NOT NULL,
+  destination_name TEXT NOT NULL,
+  delivery_unit TEXT NOT NULL,
+  transaction_id TEXT,
+  source_position TEXT,
+  delivered_at TEXT NOT NULL,
+
+  UNIQUE (idempotency_key)
+);
+
+CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source
+ON delivered_envelopes(source_name, slot_name);
+
+CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_destination
+ON delivered_envelopes(destination_name);
+
+CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source_position
+ON delivered_envelopes(source_position);

data/lib/mammoth/transaction_envelope_serializer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+
+module Mammoth
+  # Serializes CDC transaction envelopes into webhook payloads.
+  #
+  # Mammoth uses transaction envelopes as the safest delivery and checkpointing
+  # boundary for concurrent delivery. A transaction payload preserves the commit
+  # position and groups the row-level changes that belong to the same database
+  # transaction.
+  class TransactionEnvelopeSerializer
+    # Default payload type for transaction webhook delivery.
+    PAYLOAD_TYPE = "transaction.committed"
+
+    # Serialize a CDC::Core::TransactionEnvelope-like object into a Hash.
+    #
+    # @param envelope [#events, #transaction_id] transaction envelope
+    # @return [Hash] webhook-ready transaction payload
+    def self.call(envelope)
+      new(envelope).call
+    end
+
+    # @param envelope [#events, #transaction_id] transaction envelope
+    def initialize(envelope)
+      @envelope = envelope
+    end
+
+    # Return the webhook payload.
+    #
+    # @return [Hash] transaction webhook payload
+    def call
+      event_payloads = envelope.events.map { |event| EventSerializer.call(event) }
+      {
+        "event_id" => envelope_value("event_id") || SecureRandom.uuid,
+        "type" => PAYLOAD_TYPE,
+        "source" => first_event_value(event_payloads, "source") || EventSerializer::DEFAULT_SOURCE,
+        "transaction_id" => envelope.transaction_id,
+        "source_position" => source_position(event_payloads),
+        "commit_lsn" => source_position(event_payloads),
+        "committed_at" => committed_at,
+        "event_count" => event_payloads.length,
+        "events" => event_payloads,
+        "metadata" => envelope_value("metadata") || {}
+      }
+    end
+
+    # Return JSON representation of the transaction payload.
+    #
+    # @return [String] JSON representation
+    def to_json(*_args)
+      JSON.generate(call)
+    end
+
+    private
+
+    attr_reader :envelope
+
+    def envelope_hash
+      @envelope_hash ||= envelope.respond_to?(:to_h) ? stringify_keys(envelope.to_h) : {}
+    end
+
+    def stringify_keys(hash)
+      hash.to_h.transform_keys(&:to_s)
+    end
+
+    def envelope_value(key)
+      return envelope.public_send(key) if envelope.respond_to?(key)
+
+      envelope_hash[key]
+    end
+
+    def source_position(event_payloads)
+      envelope_value("source_position") || envelope_value("commit_lsn") ||
+        first_event_value(event_payloads.reverse, "source_position")
+    end
+
+    def committed_at
+      value = envelope_value("committed_at") || envelope_value("commit_time")
+      return value.iso8601 if value.respond_to?(:iso8601)
+
+      value || Time.now.utc.iso8601
+    end
+
+    def first_event_value(event_payloads, key)
+      event_payloads.find { |payload| payload[key] }&.fetch(key)
+    end
+  end
+end

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/lib/mammoth/webhook_sink.rb

@@ -2,7 +2,9 @@
 
 require "json"
 require "net/http"
+require "openssl"
 require "socket"
+require "time"
 require "uri"
 
 module Mammoth
@@ -11,27 +13,74 @@ module Mammoth
     # HTTP status range treated as successful webhook delivery.
     SUCCESS_RANGE = 200..299
 
-    attr_reader :name, :url, :timeout_seconds
+    # Supported webhook signing algorithm.
+    SIGNING_ALGORITHM = "hmac_sha256"
+    # Prefix added to generated webhook signatures.
+    SIGNATURE_PREFIX = "sha256="
+
+    attr_reader :name, :url, :timeout_seconds, :headers, :signing
 
     # @param name [String] destination name
     # @param url [String] webhook endpoint URL
     # @param timeout_seconds [Integer] HTTP open/read timeout in seconds
-    def initialize(name:, url:, timeout_seconds: 5)
+    # @param headers [Hash] static HTTP headers applied to every request
+    # @param signing [Hash, nil] HMAC signing configuration
+    def initialize(name:, url:, timeout_seconds: 5, headers: {}, signing: nil)
       @name = name
       @url = URI(url)
       @timeout_seconds = timeout_seconds
+      @headers = headers.transform_keys(&:to_s)
+      @signing = signing
     end
 
-    # Build a sink from Mammoth configuration.
-    #
-    # @param config [Mammoth::Configuration] loaded configuration
-    # @return [Mammoth::WebhookSink]
-    def self.from_config(config)
-      new(
-        name: config.dig("webhook", "name"),
-        url: config.dig("webhook", "url"),
-        timeout_seconds: config.dig("webhook", "timeout_seconds")
-      )
+    class << self
+      # Build a sink from Mammoth configuration.
+      #
+      # @param config [Mammoth::Configuration] loaded configuration
+      # @return [Mammoth::WebhookSink]
+      def from_config(config)
+        new(
+          name: config.dig("webhook", "name"),
+          url: config.dig("webhook", "url"),
+          timeout_seconds: config.dig("webhook", "timeout_seconds"),
+          headers: configured_headers(config),
+          signing: configured_signing(config)
+        )
+      end
+
+      private
+
+      def configured_headers(config)
+        static_headers = config.dig("webhook", "headers") || {}
+        env_headers = config.dig("webhook", "header_env") || {}
+
+        static_headers.merge(resolve_env_headers(env_headers))
+      end
+
+      def resolve_env_headers(env_headers)
+        env_headers.each_with_object(Hash.new) do |(header, env_name), resolved| # rubocop:disable Style/EmptyLiteral
+          resolved[header] = ENV.fetch(env_name) do
+            raise ConfigurationError, "webhook.header_env.#{header} references missing environment variable #{env_name}"
+          end
+        end
+      end
+
+      def configured_signing(config)
+        signing = config.dig("webhook", "signing")
+        return unless signing
+
+        algorithm = signing.fetch("algorithm", SIGNING_ALGORITHM)
+        raise ConfigurationError, "webhook.signing.algorithm must be #{SIGNING_ALGORITHM}" unless algorithm == SIGNING_ALGORITHM
+
+        secret_env = signing.fetch("secret_env")
+        {
+          secret: ENV.fetch(secret_env) do
+            raise ConfigurationError, "webhook.signing.secret_env references missing environment variable #{secret_env}"
+          end,
+          signature_header: signing.fetch("signature_header", "X-Mammoth-Signature"),
+          timestamp_header: signing.fetch("timestamp_header", "X-Mammoth-Timestamp")
+        }
+      end
     end
 
     # Deliver an event to the webhook endpoint.
@@ -40,7 +89,21 @@ module Mammoth
     # @return [Hash] delivery result
     # @raise [Mammoth::DeliveryError] when delivery fails
     def deliver(event)
-      payload = EventSerializer.call(event)
+      deliver_payload(EventSerializer.call(event))
+    end
+
+    # Deliver a transaction envelope to the webhook endpoint.
+    #
+    # @param envelope [#events, #transaction_id] CDC transaction envelope
+    # @return [Hash] delivery result
+    # @raise [Mammoth::DeliveryError] when delivery fails
+    def deliver_transaction(envelope)
+      deliver_payload(TransactionEnvelopeSerializer.call(envelope))
+    end
+
+    private
+
+    def deliver_payload(payload)
       response = perform_request(payload)
       return delivery_result(payload, response) if SUCCESS_RANGE.cover?(response.code.to_i)
 
@@ -49,8 +112,6 @@ module Mammoth
       raise DeliveryError, "webhook #{name} delivery failed: #{e.message}"
     end
 
-    private
-
     def perform_request(payload)
       Net::HTTP.start(url.host, url.port, use_ssl: url.scheme == "https", open_timeout: timeout_seconds,
                                           read_timeout: timeout_seconds) do |http|
@@ -59,19 +120,36 @@ module Mammoth
     end
 
     def build_request(payload)
+      body = JSON.generate(payload)
       Net::HTTP::Post.new(url.request_uri).tap do |request|
         request["Content-Type"] = "application/json"
-        request.body = JSON.generate(payload)
+        headers.each { |header, value| request[header] = value }
+        apply_signature_headers(request, body)
+        request.body = body
       end
     end
 
     def delivery_result(payload, response)
       {
         event_id: payload.fetch("event_id"),
+        payload_type: payload["type"] || "event",
         destination: name,
         status: "delivered",
         http_status: response.code.to_i
       }
     end
+
+    def apply_signature_headers(request, body)
+      return unless signing
+
+      timestamp = Time.now.utc.iso8601
+      request[signing.fetch(:timestamp_header)] = timestamp
+      request[signing.fetch(:signature_header)] = signed_body(timestamp, body)
+    end
+
+    def signed_body(timestamp, body)
+      digest = OpenSSL::HMAC.hexdigest("SHA256", signing.fetch(:secret), "#{timestamp}.#{body}")
+      "#{SIGNATURE_PREFIX}#{digest}"
+    end
   end
 end

data/lib/mammoth.rb

@@ -7,13 +7,18 @@ require_relative "mammoth/status"
 require_relative "mammoth/sqlite_store"
 require_relative "mammoth/checkpoint_store"
 require_relative "mammoth/dead_letter_store"
+require_relative "mammoth/delivered_envelope_store"
 require_relative "mammoth/event_serializer"
+require_relative "mammoth/transaction_envelope_serializer"
 require_relative "mammoth/webhook_sink"
 require_relative "mammoth/delivery_worker"
+require_relative "mammoth/delivery_processor"
+require_relative "mammoth/concurrent_delivery_runtime"
 require_relative "mammoth/sources/postgres"
 require_relative "mammoth/cdc_source"
 require_relative "mammoth/replication_consumer"
 require_relative "mammoth/application"
+require_relative "mammoth/dead_letter_commands"
 require_relative "mammoth/cli"
 
 # Mammoth is a self-hosted PostgreSQL event relay.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mammoth
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -9,6 +9,20 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: cdc-concurrent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: cdc-core
   requirement: !ruby/object:Gem::Requirement
@@ -134,8 +148,12 @@ files:
 - lib/mammoth/cdc_source.rb
 - lib/mammoth/checkpoint_store.rb
 - lib/mammoth/cli.rb
+- lib/mammoth/concurrent_delivery_runtime.rb
 - lib/mammoth/configuration.rb
+- lib/mammoth/dead_letter_commands.rb
 - lib/mammoth/dead_letter_store.rb
+- lib/mammoth/delivered_envelope_store.rb
+- lib/mammoth/delivery_processor.rb
 - lib/mammoth/delivery_worker.rb
 - lib/mammoth/errors.rb
 - lib/mammoth/event_serializer.rb
@@ -144,6 +162,7 @@ files:
 - lib/mammoth/sql/__bootstrap__.sql
 - lib/mammoth/sqlite_store.rb
 - lib/mammoth/status.rb
+- lib/mammoth/transaction_envelope_serializer.rb
 - lib/mammoth/version.rb
 - lib/mammoth/webhook_sink.rb
 homepage: https://kanutocd.github.io/mammoth/