mammoth 0.9.0 → 1.0.0

data/lib/mammoth/sources/postgres_publication_inspector.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Mammoth
+  module Sources
+    # Immutable replica-identity facts for one table in configured publications.
+    class PostgresPublicationTable
+      attr_reader :relation_id, :schema_name, :table_name, :publishes_updates, :publishes_deletes, :replica_identity,
+                  :replica_identity_columns, :primary_key_usable, :replica_identity_index_usable
+
+      # @return [void]
+      def initialize(relation_id:, schema_name:, table_name:, publishes_updates:, publishes_deletes:,
+                     replica_identity:, replica_identity_columns:, primary_key_usable:,
+                     replica_identity_index_usable:)
+        @relation_id = relation_id
+        @schema_name = schema_name
+        @table_name = table_name
+        @publishes_updates = publishes_updates
+        @publishes_deletes = publishes_deletes
+        @replica_identity = replica_identity
+        @replica_identity_columns = replica_identity_columns.freeze
+        @primary_key_usable = primary_key_usable
+        @replica_identity_index_usable = replica_identity_index_usable
+        freeze
+      end
+
+      # Schema-qualified table name for diagnostics.
+      #
+      # @return [String]
+      def qualified_name
+        "#{schema_name}.#{table_name}"
+      end
+
+      # Publication actions that require old-row identity.
+      #
+      # @return [Array<String>]
+      def identity_actions
+        actions = [] # : Array[String]
+        actions.tap do
+          actions << "UPDATE" if publishes_updates
+          actions << "DELETE" if publishes_deletes
+        end
+      end
+
+      # Whether any configured publication requires replica identity.
+      #
+      # @return [Boolean]
+      def identity_required?
+        publishes_updates || publishes_deletes
+      end
+
+      # Whether PostgreSQL can identify old rows for the published actions.
+      #
+      # `d` uses a primary key, `i` uses a selected replica-identity index,
+      # and `f` logs the full old row. `n` provides no old-row identity.
+      #
+      # @return [Boolean]
+      def identity_usable?
+        return true unless identity_required?
+
+        case replica_identity
+        when "d" then primary_key_usable
+        when "i" then replica_identity_index_usable
+        when "f" then true
+        else false
+        end
+      end
+    end
+
+    # Read-only PostgreSQL publication and replica-identity catalog inspector.
+    class PostgresPublicationInspector
+      QUERY = <<~SQL
+        SELECT
+          relation.oid AS relation_id,
+          namespace.nspname AS schema_name,
+          relation.relname AS table_name,
+          bool_or(publication.pubupdate) AS publishes_updates,
+          bool_or(publication.pubdelete) AS publishes_deletes,
+          relation.relreplident AS replica_identity,
+          CASE relation.relreplident
+            WHEN 'd' THEN (
+              SELECT COALESCE(jsonb_agg(attribute.attname ORDER BY key.ordinality), '[]'::jsonb)
+              FROM pg_index AS identity_columns_index
+              JOIN LATERAL unnest(identity_columns_index.indkey)
+                WITH ORDINALITY AS key(attribute_number, ordinality) ON true
+              JOIN pg_attribute AS attribute
+                ON attribute.attrelid = relation.oid
+               AND attribute.attnum = key.attribute_number
+              WHERE identity_columns_index.indrelid = relation.oid
+                AND identity_columns_index.indisprimary
+                AND identity_columns_index.indisvalid
+                AND identity_columns_index.indisready
+                AND identity_columns_index.indislive
+            )
+            WHEN 'i' THEN (
+              SELECT COALESCE(jsonb_agg(attribute.attname ORDER BY key.ordinality), '[]'::jsonb)
+              FROM pg_index AS identity_columns_index
+              JOIN LATERAL unnest(identity_columns_index.indkey)
+                WITH ORDINALITY AS key(attribute_number, ordinality) ON true
+              JOIN pg_attribute AS attribute
+                ON attribute.attrelid = relation.oid
+               AND attribute.attnum = key.attribute_number
+              WHERE identity_columns_index.indrelid = relation.oid
+                AND identity_columns_index.indisreplident
+                AND identity_columns_index.indisvalid
+                AND identity_columns_index.indisready
+                AND identity_columns_index.indislive
+            )
+            WHEN 'f' THEN (
+              SELECT COALESCE(jsonb_agg(attribute.attname ORDER BY attribute.attnum), '[]'::jsonb)
+              FROM pg_attribute AS attribute
+              WHERE attribute.attrelid = relation.oid
+                AND attribute.attnum > 0
+                AND NOT attribute.attisdropped
+            )
+            ELSE '[]'::jsonb
+          END AS replica_identity_columns,
+          EXISTS (
+            SELECT 1
+            FROM pg_index AS primary_index
+            WHERE primary_index.indrelid = relation.oid
+              AND primary_index.indisprimary
+              AND primary_index.indisvalid
+              AND primary_index.indisready
+              AND primary_index.indislive
+          ) AS primary_key_usable,
+          EXISTS (
+            SELECT 1
+            FROM pg_index AS identity_index
+            WHERE identity_index.indrelid = relation.oid
+              AND identity_index.indisreplident
+              AND identity_index.indisvalid
+              AND identity_index.indisready
+              AND identity_index.indislive
+          ) AS replica_identity_index_usable
+        FROM pg_publication_tables AS publication_table
+        JOIN pg_publication AS publication
+          ON publication.pubname = publication_table.pubname
+        JOIN pg_namespace AS namespace
+          ON namespace.nspname = publication_table.schemaname
+        JOIN pg_class AS relation
+          ON relation.relnamespace = namespace.oid
+         AND relation.relname = publication_table.tablename
+        WHERE publication.pubname IN (
+          SELECT jsonb_array_elements_text($1::jsonb)
+        )
+        GROUP BY namespace.nspname, relation.relname, relation.oid, relation.relreplident
+        ORDER BY namespace.nspname, relation.relname
+      SQL
+
+      attr_reader :database_url
+
+      # @param database_url [String] PostgreSQL connection URL
+      # @param connection_factory [#call, nil] optional connection factory for tests
+      def initialize(database_url:, connection_factory: nil)
+        @database_url = database_url
+        @connection_factory = connection_factory
+      end
+
+      # Inspect tables included by the configured publications.
+      #
+      # @param publication_names [Array<String>] configured publication names
+      # @return [Array<PostgresPublicationTable>]
+      def inspect(publication_names)
+        connection = open_connection
+        rows = connection.exec_params(QUERY, [JSON.generate(publication_names)])
+        rows.map { |row| build_table(row) }
+      rescue pg_error_class => e
+        raise ReplicationError, "PostgreSQL publication inspection failed: #{e.message}"
+      ensure
+        connection&.close unless connection&.finished?
+      end
+
+      private
+
+      def open_connection
+        return @connection_factory.call(database_url) if @connection_factory
+
+        require "pg"
+        PG.connect(database_url)
+      end
+
+      def build_table(row)
+        PostgresPublicationTable.new(
+          relation_id: Integer(row.fetch("relation_id")),
+          schema_name: row.fetch("schema_name"),
+          table_name: row.fetch("table_name"),
+          publishes_updates: postgres_boolean(row.fetch("publishes_updates")),
+          publishes_deletes: postgres_boolean(row.fetch("publishes_deletes")),
+          replica_identity: row.fetch("replica_identity"),
+          replica_identity_columns: parse_columns(row.fetch("replica_identity_columns")),
+          primary_key_usable: postgres_boolean(row.fetch("primary_key_usable")),
+          replica_identity_index_usable: postgres_boolean(row.fetch("replica_identity_index_usable"))
+        )
+      end
+
+      def parse_columns(value)
+        columns = value.is_a?(String) ? JSON.parse(value) : value
+        column_list = Array(columns) # : Array[untyped]
+        column_list.map(&:to_s)
+      end
+
+      def postgres_boolean(value)
+        [true, "t"].include?(value)
+      end
+
+      def pg_error_class
+        require "pg"
+        PG::Error
+      end
+    end
+  end
+end

data/lib/mammoth/sources/postgres_slot_health.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Mammoth
+  module Sources
+    # Immutable operator-facing health snapshot for one PostgreSQL replication slot.
+    #
+    # This is a Mammoth-owned policy view. The transport layer supplies catalog
+    # facts; Mammoth decides whether they make the configured source ready for
+    # continuous delivery.
+    # Typed PostgreSQL slot health and readiness policy.
+    class PostgresSlotHealth
+      attr_reader :slot_name, :present, :active, :retained_wal_bytes, :wal_status, :safe_wal_size, :inactive_since,
+                  :invalidation_reason, :restart_lsn, :restart_lsn_bytes, :confirmed_flush_lsn,
+                  :confirmed_flush_lsn_bytes, :conflicting
+
+      # Build a PostgreSQL slot health snapshot from normalized catalog facts.
+      #
+      # @return [void]
+      def initialize(slot_name:, present:, active:, retained_wal_bytes:, wal_status:, safe_wal_size:, inactive_since:,
+                     invalidation_reason:, restart_lsn:, restart_lsn_bytes:, confirmed_flush_lsn:,
+                     confirmed_flush_lsn_bytes:, conflicting:)
+        @slot_name = slot_name
+        @present = present
+        @active = active
+        @retained_wal_bytes = retained_wal_bytes
+        @wal_status = wal_status
+        @safe_wal_size = safe_wal_size
+        @inactive_since = inactive_since
+        @invalidation_reason = invalidation_reason
+        @restart_lsn = restart_lsn
+        @restart_lsn_bytes = restart_lsn_bytes
+        @confirmed_flush_lsn = confirmed_flush_lsn
+        @confirmed_flush_lsn_bytes = confirmed_flush_lsn_bytes
+        @conflicting = conflicting
+        freeze
+      end
+
+      # Build a snapshot for a missing configured slot.
+      #
+      # @param slot_name [String] configured replication slot
+      # @return [PostgresSlotHealth]
+      def self.missing(slot_name)
+        new(
+          slot_name:, present: false, active: false, retained_wal_bytes: nil,
+          wal_status: nil, safe_wal_size: nil, inactive_since: nil,
+          invalidation_reason: nil, restart_lsn: nil, restart_lsn_bytes: nil,
+          confirmed_flush_lsn: nil, confirmed_flush_lsn_bytes: nil, conflicting: false
+        )
+      end
+
+      # Whether the slot is present, active, and retaining usable WAL.
+      #
+      # @return [Boolean]
+      def ready?
+        reason.nil?
+      end
+
+      # Operator-facing reason the slot is not ready.
+      #
+      # @return [String, nil]
+      def reason
+        return "slot is missing" unless present
+        return "slot is inactive" unless active
+        return "wal_status=#{wal_status}" if %w[lost unreserved].include?(wal_status)
+        return "slot is invalidated: #{invalidation_reason}" unless blank?(invalidation_reason)
+        return "slot is conflicting" if conflicting
+        return "slot has no restart LSN" if blank?(restart_lsn)
+
+        nil
+      end
+
+      # Stable readiness payload without transport-library types.
+      #
+      # @return [Hash]
+      def summary
+        {
+          slot_name:, present:, active:, retained_wal_bytes:, wal_status:, safe_wal_size:, inactive_since:,
+          invalidation_reason:, restart_lsn:, restart_lsn_bytes:, confirmed_flush_lsn:,
+          confirmed_flush_lsn_bytes:, conflicting:, ready: ready?, reason:
+        }
+      end
+
+      private
+
+      def blank?(value)
+        value.nil? || value == ""
+      end
+    end
+  end
+end

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.9.0"
+  VERSION = "1.0.0"
 end

data/lib/mammoth.rb

@@ -18,6 +18,9 @@ require_relative "mammoth/commands/deliver_sample_command"
 require_relative "mammoth/commands/dead_letters_command"
 require_relative "mammoth/dispatch_metrics"
 require_relative "mammoth/metrics_observer"
+require_relative "mammoth/sources/postgres_slot_health"
+require_relative "mammoth/sources/postgres_publication_inspector"
+require_relative "mammoth/postgres_observability_metrics"
 require_relative "mammoth/observability_metrics"
 require_relative "mammoth/observability_snapshot"
 require_relative "mammoth/observability_server"
@@ -25,6 +28,7 @@ require_relative "mammoth/sqlite_store"
 require_relative "mammoth/checkpoint_store"
 require_relative "mammoth/dead_letter_store"
 require_relative "mammoth/delivered_envelope_store"
+require_relative "mammoth/delivery_progress_coordinator"
 require_relative "mammoth/event_serializer"
 require_relative "mammoth/transaction_envelope_serializer"
 require_relative "mammoth/persisted_payload_deserializer"
@@ -54,7 +58,7 @@ require_relative "mammoth/cli"
 
 # Mammoth is a self-hosted PostgreSQL event relay.
 #
-# Mammoth 0.8.x is a single-node PostgreSQL CDC relay with webhook fanout,
+# Mammoth 1.x is a single-node PostgreSQL CDC relay with webhook fanout,
 # replayable operational SQLite state, health/metrics endpoints, and explicit
 # extension contracts for state, destination, runtime, lifecycle, configuration,
 # and local command integrations.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mammoth
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -51,34 +51,54 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: pgoutput-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: pgoutput-decoder
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: pgoutput-parser
   requirement: !ruby/object:Gem::Requirement
@@ -99,20 +119,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -181,6 +201,7 @@ files:
 - lib/mammoth/dead_letter_store.rb
 - lib/mammoth/delivered_envelope_store.rb
 - lib/mammoth/delivery_processor.rb
+- lib/mammoth/delivery_progress_coordinator.rb
 - lib/mammoth/delivery_worker.rb
 - lib/mammoth/destinations/adapter.rb
 - lib/mammoth/destinations/registry.rb
@@ -199,6 +220,7 @@ files:
 - lib/mammoth/operational_state/registry.rb
 - lib/mammoth/operational_state/sqlite_adapter.rb
 - lib/mammoth/persisted_payload_deserializer.rb
+- lib/mammoth/postgres_observability_metrics.rb
 - lib/mammoth/registry.rb
 - lib/mammoth/replication_consumer.rb
 - lib/mammoth/route_filter.rb
@@ -208,6 +230,8 @@ files:
 - lib/mammoth/runtimes/inline_adapter.rb
 - lib/mammoth/runtimes/registry.rb
 - lib/mammoth/sources/postgres.rb
+- lib/mammoth/sources/postgres_publication_inspector.rb
+- lib/mammoth/sources/postgres_slot_health.rb
 - lib/mammoth/sql/__bootstrap__.sql
 - lib/mammoth/sqlite_store.rb
 - lib/mammoth/status.rb