pgoutput-source-adapter 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab9b7c87b7ede5dd583ae9457f30fe30fdfcf1dd55b2ca33e08162d23f36601b
-  data.tar.gz: 5c7a7b1a081f8a67a07397ecd8034a462e679e636dd2a51d71b9512e90333c85
+  metadata.gz: 96f17f6324e61545be066327df21f021a9b00a1c8a77f5868d74f87ec716e5f2
+  data.tar.gz: e56b9ff8c51ca0952acf5ce00663c9fd406caf9a1e1c9439f6f58caf19c28d41
 SHA512:
-  metadata.gz: 3a19f060ba7f5552dec37471a114fa4ee30f0c960d20a27c82676f888753c53fe76c4473038170185806dda76988217ab882cf90f5186fdfa28d5dfef438c12a
-  data.tar.gz: 00f2d815d0ff8f2d8b9865ecdbee82c528fba3341ecf0125c8f32ddc930365c669bb28ad0cc3c1274c7208089a4eca9ff05bf6d47a9061b234563c92b9c3fd18
+  metadata.gz: 8c4868ad4effe15ace70dae52991e7c45f64116cbe6ea613df77d494bdfd9486a9bbcbcabcd31873d5e474655edfde50588563e498f2a11ef0e9bf9d8ab1e76e
+  data.tar.gz: de92c820e593d5d5f05e41885f002333df1aa0d16d00af1f5c4d5f05df8f8655571d8261cf05192a7807830d263db65e6e94908af61c786efd9b1a2a01053443

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 ## [Unreleased]
 
+## [0.2.0] - 2026-07-17
+
+### Added
+
+- Added incremental `Cdc#each_normalized` transaction normalization for
+  unbounded replication streams.
+- Added `Cdc#stream_event` so source integrations can preserve transport WAL
+  positions without owning transaction buffering.
+
+### Documentation
+
+- Raised and enforced generated YARD API documentation coverage at 100%.
+
+### Quality
+
+- Raised and enforced SimpleCov line and branch coverage at 100%.
+
+## [0.1.1] - 2026-06-16
+
+- Improved gemspec summary and description
+
 ## [0.1.0] - 2026-06-16
 
 ### Added
@@ -31,4 +52,4 @@
 ### Coverage
 
 - Line Coverage: 97.62%
-- Branch Coverage: 91.18%
+- Branch Coverage: 91.18%

data/README.md

@@ -85,6 +85,26 @@ envelope = results.first
 # => CDC::Core::TransactionEnvelope
 ```
 
+Normalize an unbounded replication stream incrementally:
+
+```ruby
+inputs = Enumerator.new do |stream|
+  runner.start do |payload, metadata|
+    decoded = decoder.decode(parser.process(payload))
+    stream << adapter.stream_event(decoded, source_position: metadata.wal_end_lsn)
+  end
+end
+
+adapter.each_normalized(inputs) do |work|
+  processor.process(work)
+end
+```
+
+Transaction buffering belongs to the source adapter. Changes outside a
+transaction are yielded immediately; changes between `Begin` and `Commit` are
+yielded as a `CDC::Core::TransactionEnvelope`. `normalize_many` uses the same
+normalization path for finite batches.
+
 ## Primary keys
 
 For update and delete events, pgoutput may provide an old-key tuple. When it does, that tuple is used as the `CDC::Core::ChangeEvent#primary_key`.
@@ -151,6 +171,10 @@ Those responsibilities belong to `pgoutput-client`, `pgoutput-parser`, `pgoutput
 bundle exec rake
 ```
 
+The default quality task enforces 100% line and branch coverage and 100% YARD
+API documentation coverage. Run `bundle exec yard stats --list-undoc` to inspect
+the documented public API objects.
+
 ## License
 
 [MIT](./LICENSE.txt).

data/Rakefile

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'bundler/gem_tasks'
+require 'open3'
 require 'rake/testtask'
 require 'rubocop/rake_task'
 require 'yard'
@@ -17,6 +18,16 @@ end
 
 YARD::Rake::YardocTask.new(:yard)
 
+desc 'Require complete public API documentation'
+task yard_coverage: :yard do
+  output, status = Open3.capture2e('bundle', 'exec', 'yard', 'stats', '--list-undoc')
+  puts output
+
+  next if status.success? && output.include?('100.00% documented')
+
+  abort 'YARD documentation coverage must remain at 100%'
+end
+
 desc 'Validate rbs sig files'
 task :steep do
   sh 'bundle exec steep check'
@@ -27,4 +38,4 @@ task :coverage do
   sh 'xdg-open coverage/index.html'
 end
 
-task default: %i[test rubocop steep yard]
+task default: %i[test rubocop steep yard_coverage]

data/lib/pgoutput/source/adapter.rb

@@ -2,7 +2,15 @@
 
 require_relative '../source_adapter'
 
+# Root namespace for pgoutput protocol libraries.
+#
+# @api public
 module Pgoutput
+  # Backward-compatible namespace for the original generated gem layout.
+  #
+  # New integrations should use {Pgoutput::SourceAdapter} directly.
+  #
+  # @api public
   module Source
     # Compatibility alias for {Pgoutput::SourceAdapter}.
     #

data/lib/pgoutput/source_adapter/cdc.rb

@@ -2,12 +2,20 @@
 
 require 'cdc/core'
 
-unless CDC::Core.const_defined?(:SourceAdapter)
-  raise LoadError, 'pgoutput-source-adapter requires a cdc-core version that defines CDC::Core::SourceAdapter'
-end
-
 module Pgoutput
   module SourceAdapter
+    # Resolves the required cdc-core adapter contract.
+    #
+    # @api private
+    def self.ensure_core_source_adapter!(core)
+      return if core.const_defined?(:SourceAdapter)
+
+      raise LoadError, 'pgoutput-source-adapter requires a cdc-core version that defines CDC::Core::SourceAdapter'
+    end
+    private_class_method :ensure_core_source_adapter!
+
+    ensure_core_source_adapter!(CDC::Core)
+
     # Normalizes Pgoutput::Decoder::Events into CDC::Core primitives.
     #
     # This adapter is intentionally located under the Pgoutput namespace because
@@ -22,10 +30,40 @@ module Pgoutput
     # @example Normalize a decoded transaction event batch
     #   envelope = adapter.normalize_many([begin_event, insert_event, commit_event]).first
     #
+    # @example Normalize an unbounded decoded event stream
+    #   adapter.each_normalized(decoded_events) { |work| process(work) }
+    #
     # @api public
+    # rubocop:disable Metrics/ClassLength
     class Cdc < CDC::Core::SourceAdapter
+      # Canonical source name included in normalized event metadata.
+      #
+      # @return [String]
       SOURCE_NAME = 'pgoutput'
 
+      # A decoded event paired with its transport-level source position.
+      #
+      # Source integrations use this value object to preserve WAL position
+      # metadata without teaching downstream applications about pgoutput
+      # transaction boundaries.
+      #
+      # @api public
+      class StreamEvent
+        # @return [Object] decoded pgoutput event
+        attr_reader :event
+        # @return [String, nil] transport WAL position
+        attr_reader :source_position
+
+        # @param event [Object] decoded pgoutput event
+        # @param source_position [String, nil] transport WAL position
+        # @return [void]
+        def initialize(event, source_position)
+          @event = event
+          @source_position = source_position
+          freeze
+        end
+      end
+
       # @param primary_key_resolver [#call, nil] optional callable used to infer
       #   primary keys from decoded row values when pgoutput does not provide an
       #   old-key tuple. The callable receives the decoded event and value hash.
@@ -50,7 +88,7 @@ module Pgoutput
       # @raise [Pgoutput::SourceAdapter::Error] when the decoded event type is
       #   unsupported.
       # rubocop:disable Metrics/MethodLength
-      def normalize(event)
+      def normalize(event, source_position: nil)
         case event_name(event)
         when 'Insert'
           change_event(
@@ -58,7 +96,8 @@ module Pgoutput
             operation: :insert,
             old_values: nil,
             new_values: event.values,
-            primary_key: primary_key_for(event, event.values)
+            primary_key: primary_key_for(event, event.values),
+            source_position: source_position
           )
         when 'Update'
           old_values = event.old_values || event.old_key
@@ -68,7 +107,8 @@ module Pgoutput
             operation: :update,
             old_values: old_values,
             new_values: event.new_values,
-            primary_key: primary_key_for(event, event.new_values)
+            primary_key: primary_key_for(event, event.new_values),
+            source_position: source_position
           )
         when 'Delete'
           old_values = event.old_values || event.old_key
@@ -78,7 +118,8 @@ module Pgoutput
             operation: :delete,
             old_values: old_values,
             new_values: nil,
-            primary_key: primary_key_for(event, old_values)
+            primary_key: primary_key_for(event, old_values),
+            source_position: source_position
           )
         when 'Begin', 'Commit'
           nil
@@ -88,6 +129,31 @@ module Pgoutput
       end
       # rubocop:enable Metrics/MethodLength
 
+      # Pair a decoded event with the source position supplied by its transport.
+      #
+      # @param event [Object] decoded pgoutput event
+      # @param source_position [Object, nil] WAL position associated with the event
+      # @return [StreamEvent] immutable streaming input
+      def stream_event(event, source_position: nil)
+        StreamEvent.new(event, lsn_string(source_position))
+      end
+
+      # Incrementally normalize a decoded pgoutput event stream.
+      #
+      # Transaction buffering belongs to this source adapter: row changes are
+      # retained after Begin and emitted as one CDC::Core::TransactionEnvelope
+      # at Commit. Changes outside a transaction are yielded immediately.
+      #
+      # @param events [Enumerable<Object, StreamEvent>] decoded stream
+      # @yieldparam work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
+      # @return [Enumerator, nil] enumerator without a block, otherwise nil
+      def each_normalized(events, &block)
+        return enum_for(:each_normalized, events) unless block
+
+        normalize_stream(events, flush_incomplete: false, &block)
+        nil
+      end
+
       # Normalize a sequence of decoded pgoutput events.
       #
       # If the sequence contains transaction boundaries, row changes between a
@@ -99,17 +165,28 @@ module Pgoutput
       #   normalized row changes and transaction envelopes in input order.
       # @raise [Pgoutput::SourceAdapter::Error] when any decoded event type is
       #   unsupported.
+      def normalize_many(events)
+        results = [] # : Array[CDC::Core::ChangeEvent | CDC::Core::TransactionEnvelope]
+        normalize_stream(events, flush_incomplete: true) { |work| results << work }
+        share(results.freeze)
+      end
+
+      private
+
+      attr_reader :primary_key_resolver, :metadata_builder
+
       # rubocop:disable Metrics/AbcSize
       # rubocop:disable Metrics/PerceivedComplexity
       # rubocop:disable Metrics/CyclomaticComplexity
       # rubocop:disable Metrics/MethodLength
-      def normalize_many(events)
-        results = [] # : Array[CDC::Core::ChangeEvent | CDC::Core::TransactionEnvelope]
+      def normalize_stream(events, flush_incomplete:, &block)
         transaction_id = nil
         transaction_events = [] # : Array[CDC::Core::ChangeEvent]
         transaction_metadata = {} # : Hash[String, untyped]
 
-        events.each do |event|
+        # rubocop:disable Metrics/BlockLength
+        events.each do |input|
+          event, source_position = stream_input(input)
           case event_name(event)
           when 'Begin'
             transaction_id = event.transaction_id
@@ -120,11 +197,14 @@ module Pgoutput
             )
           when 'Commit'
             if transaction_id || !transaction_events.empty?
-              results << transaction_envelope(
-                event,
-                transaction_id: transaction_id || event.transaction_id,
-                events: transaction_events,
-                metadata: transaction_metadata
+              block.call(
+                transaction_envelope(
+                  event,
+                  transaction_id: transaction_id || event.transaction_id,
+                  events: transaction_events,
+                  metadata: transaction_metadata,
+                  source_position: source_position
+                )
               )
             end
 
@@ -132,30 +212,26 @@ module Pgoutput
             transaction_events = []
             transaction_metadata = {}
           else
-            normalized = normalize(event)
+            normalized = normalize(event, source_position: source_position)
             next if normalized.nil?
 
             if transaction_id
               transaction_events << normalized
             else
-              results << normalized
+              block.call(normalized)
             end
           end
         end
+        # rubocop:enable Metrics/BlockLength
 
-        results.concat(transaction_events) if transaction_id && !transaction_events.empty?
-        share(results.freeze)
+        transaction_events.each(&block) if flush_incomplete && transaction_id && !transaction_events.empty?
       end
       # rubocop:enable Metrics/AbcSize
       # rubocop:enable Metrics/PerceivedComplexity
       # rubocop:enable Metrics/CyclomaticComplexity
       # rubocop:enable Metrics/MethodLength
 
-      private
-
-      attr_reader :primary_key_resolver, :metadata_builder
-
-      def change_event(event, operation:, old_values:, new_values:, primary_key:)
+      def change_event(event, operation:, old_values:, new_values:, primary_key:, source_position:)
         share(
           CDC::Core::ChangeEvent.new(
             operation: operation,
@@ -165,17 +241,18 @@ module Pgoutput
             new_values: new_values,
             primary_key: primary_key,
             transaction_id: event.transaction_id,
+            commit_lsn: lsn_string(source_position),
             metadata: metadata_for(event)
           )
         )
       end
 
-      def transaction_envelope(event, transaction_id:, events:, metadata:)
+      def transaction_envelope(event, transaction_id:, events:, metadata:, source_position: nil)
         share(
           CDC::Core::TransactionEnvelope.new(
             transaction_id: transaction_id,
             events: events.freeze,
-            commit_lsn: lsn_string(event.commit_lsn),
+            commit_lsn: lsn_string(event.commit_lsn || source_position),
             committed_at: event.commit_timestamp,
             metadata: metadata.merge(metadata_for(event)).merge(
               'commit_flags' => event.flags,
@@ -191,6 +268,12 @@ module Pgoutput
         primary_key_resolver.call(event, values)
       end
 
+      def stream_input(input)
+        return [input.event, input.source_position] if input.is_a?(StreamEvent)
+
+        [input, nil]
+      end
+
       def default_primary_key(_event, values)
         return nil unless values.respond_to?(:key?)
 
@@ -232,11 +315,12 @@ module Pgoutput
         hash.compact
       end
 
-      def share(object)
-        Ractor.make_shareable(object)
+      def share(object, ractor: Ractor)
+        ractor.make_shareable(object)
       rescue Ractor::Error
         object
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/pgoutput/source_adapter/version.rb

@@ -5,6 +5,6 @@ module Pgoutput
     # Current pgoutput-source-adapter gem version.
     #
     # @return [String] semantic version published to RubyGems.
-    VERSION = '0.1.1'
+    VERSION = '0.2.0'
   end
 end

data/sig/pgoutput/source_adapter/cdc.rbs

@@ -1,5 +1,7 @@
 module Pgoutput
   module SourceAdapter
+    def self.ensure_core_source_adapter!: (untyped core) -> nil
+
     # Normalizes Pgoutput::Decoder::Events into CDC::Core primitives.
     #
     # This adapter is intentionally located under the Pgoutput namespace because
@@ -22,6 +24,13 @@ module Pgoutput
 
       SOURCE_NAME: "pgoutput"
 
+      class StreamEvent
+        attr_reader event: untyped
+        attr_reader source_position: untyped
+
+        def initialize: (untyped event, untyped source_position) -> void
+      end
+
       # @param primary_key_resolver [#call, nil] optional callable used to infer
       #   primary keys from decoded row values when pgoutput does not provide an
       #   old-key tuple.
@@ -37,7 +46,13 @@ module Pgoutput
       #
       # @param event [Object] a Pgoutput::Decoder::Events object.
       # @return [CDC::Core::ChangeEvent, nil]
-      def normalize: (untyped event) -> untyped
+      def normalize: (untyped event, ?source_position: untyped) -> untyped
+
+      # Pair a decoded event with its transport-level source position.
+      def stream_event: (untyped event, ?source_position: untyped) -> StreamEvent
+
+      # Incrementally normalize an unbounded decoded event stream.
+      def each_normalized: (untyped events) ?{ (untyped work) -> untyped } -> (Enumerator[untyped, nil] | nil)
 
       # Normalize a sequence of decoded pgoutput events.
       #
@@ -55,12 +70,16 @@ module Pgoutput
 
       attr_reader metadata_builder: untyped
 
-      def change_event: (untyped event, operation: untyped, old_values: untyped, new_values: untyped, primary_key: untyped) -> untyped
+      def normalize_stream: (untyped events, flush_incomplete: bool) { (untyped work) -> untyped } -> void
+
+      def change_event: (untyped event, operation: untyped, old_values: untyped, new_values: untyped, primary_key: untyped, source_position: untyped) -> untyped
 
-      def transaction_envelope: (untyped event, transaction_id: untyped, events: untyped, metadata: untyped) -> untyped
+      def transaction_envelope: (untyped event, transaction_id: untyped, events: untyped, metadata: untyped, ?source_position: untyped) -> untyped
 
       def primary_key_for: (untyped event, untyped values) -> untyped
 
+      def stream_input: (untyped input) -> [untyped, untyped]
+
       def default_primary_key: (untyped _event, untyped values) -> (nil | untyped)
 
       def metadata_for: (untyped event) -> untyped

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgoutput-source-adapter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Ken C. Demanawa