mammoth 0.7.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f6df928a12cd055b0c90ed4cd90ae18dca55dcfe43d2c990679543c0be5bcb8
-  data.tar.gz: e87e0265edd19a481ff0a49921e8116f86d005618d585b6529e8a99c1505795a
+  metadata.gz: 6bca055c346ee0a0b651861f60fca15910708ea772b1cfb661cbc8a46040a1e0
+  data.tar.gz: 03d9953cec8a9d82859cbfe446736b4cdc216d46e36244bb5fca4abe455fb0ef
 SHA512:
-  metadata.gz: 4e12848d7f5590461f9e02a34e9b8917719e49347f86e489e927fb88df26964e7e7363fca444224cc1826f62ce0ecfea70880e2d1aae55ea766d5dbdde231282
-  data.tar.gz: f23a605b1bfbb1280abcd473d658393516c57a90a4c571ee2dd10eaeb84415f131b021471cf913c9f073a209e23e077c709c234829d8533b5379169f1869d4a0
+  metadata.gz: 3394c678ad2135d34d9d5429599d33d02b8390d7d366821f71356fb0ed50bee71892594901372b0083646f4b8f2332e80977933651102c2e72c3a9309770bee2
+  data.tar.gz: 3b67803444f90aa6951f6c554d70b0d7b6e88d7b020a905edbda6357a3f54a7eacbd5b3087a66fd1c002f740596cdcf9515f5ae565e6eb7f00cf66ecba648791

data/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## Unreleased
 
+## 0.8.0
+
+### Changed
+
+- Delegated incremental PostgreSQL transaction buffering and
+  `CDC::Core::TransactionEnvelope` construction to `pgoutput-source-adapter`.
+- `Mammoth::Sources::Postgres` now streams decoded events and transport WAL
+  positions through the adapter's `each_normalized` API instead of maintaining
+  private transaction state or envelope lookalikes.
+
+### Quality
+
+- Added boundary and source integration coverage for streaming adapter
+  delegation, exact core envelope preservation, and WAL position forwarding.
+- Updated API signatures, examples, benchmark descriptions, and architecture
+  documentation for the corrected source-adapter boundary.
+
 ## 0.7.2
 
 ### Changed

data/README.md

@@ -1,3 +1,7 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/kanutocd/mammoth/main/docs/assets/logo/mammoth-horizontal.svg" alt="Mammoth" width="520">
+</p>
+
 # Mammoth
 
 [![Gem Version](https://badge.fury.io/rb/mammoth.svg)](https://badge.fury.io/rb/mammoth)
@@ -5,7 +9,7 @@
 [![Ruby Version](https://img.shields.io/badge/ruby-%3E%3D%204.0-ruby.svg)](https://www.ruby-lang.org/en/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-🦣 Mammoth is a self-hosted PostgreSQL event relay focused on reliable delivery
+Mammoth is a self-hosted PostgreSQL event relay focused on reliable delivery
 of database change events.
 
 ```text
@@ -20,7 +24,7 @@ Mammoth
 Webhook fanout
 ```
 
-🦣 Mammoth is intentionally boring infrastructure. It uses YAML configuration,
+Mammoth is intentionally boring infrastructure. It uses YAML configuration,
 JSON Schema validation, local SQLite operational state, and the CDC Ecosystem's
 shared vocabulary so operators can inspect, recover, and reason about delivery.
 
@@ -37,7 +41,7 @@ https://kanutocd.github.io/mammoth/Mammoth.html
 
 ## OSS MVP
 
-🦣 Mammoth OSS includes:
+Mammoth OSS includes:
 
 - CLI foundation
 - YAML configuration loading
@@ -69,6 +73,12 @@ Mammoth does not own pgoutput protocol parsing, value decoding, source
 normalization, ordering policy, or runtime execution. Those belong to the
 upstream CDC Ecosystem components.
 
+For the live PostgreSQL stream, `pgoutput-source-adapter` incrementally owns
+`Begin`/`Commit` buffering and emits exact `CDC::Core::ChangeEvent` or
+`CDC::Core::TransactionEnvelope` work items. Mammoth only composes the
+transport, parser, decoder, and source adapter and forwards the resulting core
+work to delivery.
+
 ## Extensions
 
 Mammoth OSS exposes small adapter registries for future extensions:

data/lib/mammoth/sources/postgres.rb

@@ -8,7 +8,7 @@ module Mammoth
     # Postgres realizes the CDC Ecosystem libraries for Mammoth's product
     # boundary. It composes pgoutput-client, pgoutput-parser,
     # pgoutput-decoder, and pgoutput-source-adapter into a single source that
-    # yields CDC::Core-shaped work to the delivery runtime.
+    # yields CDC::Core work to the delivery runtime.
     #
     # This class may mention pgoutput implementation details because it is the
     # concrete PostgreSQL source adapter used by Mammoth. The rest of Mammoth
@@ -45,22 +45,26 @@ module Mammoth
         @checkpoint_store = checkpoint_store
       end
 
-      # Stream CDC::Core-shaped work from PostgreSQL logical replication.
+      # Stream CDC::Core work from PostgreSQL logical replication.
       #
       # Calling this method starts the injected or configured pgoutput-client
       # runner. The runner owns the PostgreSQL replication connection and slot
-      # lifecycle; this class only composes the parser, decoder, and adapter
-      # libraries around the stream.
+      # lifecycle, while pgoutput-source-adapter owns transaction buffering and
+      # normalization into CDC::Core work items. This class only composes those
+      # layers and forwards transport source positions to the adapter.
       #
-      # @yieldparam work [Object] CDC::Core::ChangeEvent or TransactionEnvelope
+      # @yieldparam work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
       # @return [Enumerator, nil]
       # @raise [Mammoth::ReplicationError] when the source cannot stream CDC work
       def each(&block)
         return enum_for(:each) unless block_given?
 
-        effective_runner.start do |payload, metadata = nil|
-          process_payload(payload, metadata, &block)
+        normalizer = effective_adapter
+        unless normalizer.respond_to?(:each_normalized)
+          raise ReplicationError, "pgoutput source adapter must respond to #each_normalized"
         end
+
+        normalizer.each_normalized(decoded_stream, &block)
         nil
       rescue StandardError => e
         raise e if e.is_a?(ReplicationError)
@@ -70,41 +74,37 @@ module Mammoth
 
       private
 
+      def decoded_stream
+        Enumerator.new do |stream|
+          effective_runner.start do |payload, metadata = nil|
+            process_payload(payload, metadata) { |decoded| stream << stream_event(decoded, metadata) }
+          end
+        end
+      end
+
       def process_payload(payload, metadata, &block)
         parsed = parse_payload(payload)
         decoded = decode_message(parsed, metadata)
-        process_decoded(decoded, metadata, &block)
+        each_decoded(decoded, &block)
       end
 
-      # rubocop:disable Metrics/MethodLength
-      def process_decoded(decoded, metadata, &block)
+      def each_decoded(decoded, &block)
         return if decoded.nil?
 
         if decoded.is_a?(Array)
-          decoded.each { |item| process_decoded(item, metadata, &block) }
+          decoded.each { |item| each_decoded(item, &block) }
           return
         end
 
-        if begin_message?(decoded)
-          start_transaction_buffer(decoded)
-          return
-        end
+        block.call(decoded)
+      end
 
-        if commit_message?(decoded)
-          emit_transaction_buffer(decoded, metadata, &block)
-          return
-        end
+      def stream_event(decoded, metadata)
+        normalizer = effective_adapter
+        return decoded unless normalizer.respond_to?(:stream_event)
 
-        normalize_decoded(decoded).each do |work|
-          work = enrich_work_position(work, metadata, decoded)
-          if transaction_buffer_active?
-            Array(@transaction_events) << work
-          else
-            block.call(work)
-          end
-        end
+        normalizer.stream_event(decoded, source_position: source_position(metadata))
       end
-      # rubocop:enable Metrics/MethodLength
 
       def parse_payload(payload)
         parser = effective_parser
@@ -127,102 +127,6 @@ module Mammoth
         raise ReplicationError, "pgoutput decoder must respond to #decode or #call"
       end
 
-      def normalize_decoded(decoded)
-        return [] if decoded.nil?
-        return decoded.flat_map { |item| normalize_decoded(item) } if decoded.is_a?(Array)
-
-        adapter = effective_adapter
-        result = if adapter.respond_to?(:normalize)
-                   adapter.normalize(decoded)
-                 elsif adapter.respond_to?(:call)
-                   adapter.call(decoded)
-                 else
-                   raise ReplicationError, "pgoutput source adapter must respond to #normalize or #call"
-                 end
-
-        result.is_a?(Array) ? result.compact : [result].compact
-      end
-
-      def begin_message?(decoded)
-        message_kind(decoded).include?("begin")
-      end
-
-      def commit_message?(decoded)
-        message_kind(decoded).include?("commit")
-      end
-
-      def message_kind(decoded)
-        (value_from(decoded, :message_type, :type, :kind) || decoded.class.name.to_s.split("::").last).to_s.downcase
-      end
-
-      def start_transaction_buffer(decoded)
-        @transaction_events = []
-        @transaction_id = value_from(decoded, :transaction_id, :xid, :final_lsn)
-        @transaction_metadata = value_hash(decoded, :metadata) || {}
-      end
-
-      def emit_transaction_buffer(decoded, metadata, &block)
-        return unless transaction_buffer_active?
-
-        block.call(
-          TransactionEnvelope.new(
-            @transaction_events,
-            transaction_id_for(decoded),
-            commit_lsn_for(decoded, metadata),
-            value_from(decoded, :commit_time, :committed_at, :timestamp),
-            @transaction_metadata
-          )
-        )
-      ensure
-        clear_transaction_buffer
-      end
-
-      def transaction_id_for(decoded)
-        value_from(decoded, :transaction_id, :xid) || @transaction_id || first_event_value(:transaction_id, :xid)
-      end
-
-      def commit_lsn_for(decoded, metadata)
-        value_from(decoded, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
-          value_from(metadata, :commit_lsn, :source_position, :lsn, :end_lsn, :final_lsn) ||
-          first_event_value(:commit_lsn, :source_position, :lsn)
-      end
-
-      def transaction_buffer_active?
-        !@transaction_events.nil?
-      end
-
-      def clear_transaction_buffer
-        @transaction_events = nil
-        @transaction_id = nil
-        @transaction_metadata = nil
-      end
-
-      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      def enrich_work_position(work, metadata, decoded)
-        position = value_from(work, :source_position, :commit_lsn) ||
-                   value_from(metadata, :source_position, :commit_lsn, :lsn) ||
-                   value_from(decoded, :source_position, :commit_lsn, :lsn)
-        return work unless position
-
-        work_hash = work.respond_to?(:to_h) ? work.to_h : work
-        return work unless work_hash.is_a?(Hash)
-
-        key_style = work_hash.key?("operation") ? :string : :symbol
-        source_position_key = key_style == :string ? "source_position" : :source_position
-        commit_lsn_key = key_style == :string ? "commit_lsn" : :commit_lsn
-        return work if work_hash[source_position_key] || work_hash[commit_lsn_key]
-
-        work_hash.merge(source_position_key => position, commit_lsn_key => position)
-      end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
-      def first_event_value(*keys)
-        event = Array(@transaction_events).find do |event|
-          keys.any? { |key| value_from(event, key) }
-        end
-        value_from(event, *keys) if event
-      end
-
       def value_from(object, *keys)
         return nil if object.nil?
 
@@ -239,9 +143,8 @@ module Mammoth
         nil
       end
 
-      def value_hash(object, key)
-        value = value_from(object, key)
-        value.is_a?(Hash) ? value : nil
+      def source_position(metadata)
+        value_from(metadata, :source_position, :commit_lsn, :lsn, :wal_end_lsn, :end_lsn, :final_lsn)
       end
 
       def effective_runner
@@ -375,8 +278,6 @@ module Mammoth
       rescue LoadError => e
         raise ReplicationError, "#{gem_name} is required for PostgreSQL CDC source integration: #{e.message}"
       end
-      TransactionEnvelope = Data.define(:events, :transaction_id, :commit_lsn, :commit_time, :metadata)
-      private_constant :TransactionEnvelope
     end
     # rubocop:enable Metrics/ClassLength
   end

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.7.2"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mammoth
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -99,14 +99,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement