mammoth 0.0.0

data/lib/mammoth/errors.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Base error for all Mammoth failures.
+  class Error < StandardError; end
+
+  # Raised when configuration cannot be loaded or validated.
+  class ConfigurationError < Error; end
+
+  # Raised when local SQLite operational state cannot be initialized.
+  class StoreError < Error; end
+
+  # Raised when webhook delivery fails.
+  class DeliveryError < Error; end
+
+  # Raised when replication consumption is not available or fails.
+  class ReplicationError < Error; end
+end

data/lib/mammoth/event_serializer.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "time"
+
+module Mammoth
+  # Serializes CDC-core change events into webhook payloads.
+  #
+  # The serializer projects Mammoth's sink payload from CDC vocabulary rather
+  # than pgoutput protocol vocabulary. That keeps webhook delivery independent
+  # from PostgreSQL-specific message shapes while preserving source metadata such
+  # as commit LSN and transaction identity when available.
+  class EventSerializer
+    DEFAULT_SOURCE = "postgresql"
+
+    # Serialize an event-like object into a webhook-ready Hash.
+    #
+    # @param event [Hash, #to_h] normalized CDC event
+    # @return [Hash] webhook payload
+    def self.call(event)
+      new(event).call
+    end
+
+    # @param event [Hash, #to_h] normalized CDC event
+    def initialize(event)
+      @event = event.respond_to?(:to_h) ? event.to_h : event
+    end
+
+    # Return the webhook payload.
+    #
+    # @return [Hash] webhook payload
+    def call
+      event_hash = stringify_keys(@event)
+      {
+        "event_id" => event_hash["event_id"] || SecureRandom.uuid,
+        "source" => event_hash["source"] || DEFAULT_SOURCE,
+        "operation" => normalize_operation(event_hash.fetch("operation")),
+        "namespace" => event_hash["namespace"] || event_hash["schema"],
+        "entity" => event_hash["entity"] || event_hash["table"],
+        "identity" => event_hash["identity"] || event_hash["primary_key"],
+        "source_position" => event_hash["source_position"] || event_hash["commit_lsn"],
+        "transaction_id" => event_hash["transaction_id"],
+        "occurred_at" => occurred_at(event_hash),
+        "data" => event_data(event_hash),
+        "changes" => event_hash["changes"] || [],
+        "metadata" => event_hash["metadata"] || {}
+      }
+    end
+
+    # Return JSON representation of the webhook payload.
+    #
+    # @return [String] JSON representation of the payload
+    def to_json(*_args)
+      JSON.generate(call)
+    end
+
+    private
+
+    def stringify_keys(hash)
+      hash.to_h.transform_keys(&:to_s)
+    end
+
+    def normalize_operation(operation)
+      operation.to_s
+    end
+
+    def occurred_at(event_hash)
+      value = event_hash["occurred_at"]
+      return value.iso8601 if value.respond_to?(:iso8601)
+
+      value || Time.now.utc.iso8601
+    end
+
+    def event_data(event_hash)
+      event_hash["data"] || event_hash["new_values"] || event_hash["old_values"] || {}
+    end
+  end
+end

data/lib/mammoth/pgoutput_source.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Streams PostgreSQL logical replication through the CDC Ecosystem boundary.
+  #
+  # PgoutputSource is Mammoth's upstream integration point. It composes the
+  # standalone pgoutput transport, parser, decoder, and source-adapter gems so
+  # the rest of Mammoth only receives CDC-core domain objects. Transport
+  # resiliency remains owned by pgoutput-client; Mammoth owns delivery.
+  class PgoutputSource
+    # @return [Mammoth::Configuration] loaded Mammoth configuration
+    attr_reader :config
+    # @return [Object, nil] pgoutput-client compatible runner
+    attr_reader :runner
+    # @return [Object, nil] pgoutput-parser compatible parser
+    attr_reader :parser
+    # @return [Object, nil] pgoutput-decoder compatible decoder
+    attr_reader :decoder
+    # @return [Object, nil] CDC source adapter
+    attr_reader :source_adapter
+
+    # Build the pgoutput integration source.
+    #
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param runner [Object, nil] injectable pgoutput-client runner
+    # @param parser [Object, nil] injectable pgoutput parser
+    # @param decoder [Object, nil] injectable pgoutput decoder
+    # @param source_adapter [Object, nil] injectable CDC source adapter
+    def initialize(config, runner: nil, parser: nil, decoder: nil, source_adapter: nil)
+      @config = config
+      @runner = runner
+      @parser = parser
+      @decoder = decoder
+      @source_adapter = source_adapter
+    end
+
+    # Stream CDC-core objects from PostgreSQL.
+    #
+    # @yieldparam work [Object] CDC::Core::ChangeEvent or TransactionEnvelope
+    # @return [void]
+    # @raise [Mammoth::ReplicationError] when required CDC components are unavailable
+    def each
+      return enum_for(:each) unless block_given?
+
+      effective_runner.start do |payload, metadata|
+        normalized_items(payload, metadata).each { |item| yield item }
+      end
+    end
+
+    private
+
+    def normalized_items(payload, metadata)
+      decoded = effective_decoder ? invoke_component(effective_decoder, parsed_payload(payload), metadata) : parsed_payload(payload)
+      normalized = invoke_source_adapter(decoded, metadata)
+      Array(normalized).flatten
+    end
+
+    def parsed_payload(payload)
+      return payload unless effective_parser
+
+      invoke_component(effective_parser, payload)
+    end
+
+    def invoke_source_adapter(decoded, metadata)
+      adapter = effective_source_adapter
+      if adapter.respond_to?(:normalize)
+        adapter.normalize(decoded)
+      elsif adapter.respond_to?(:call)
+        adapter.call(decoded, metadata)
+      else
+        raise ReplicationError, "pgoutput source adapter must respond to #normalize or #call"
+      end
+    end
+
+    def invoke_component(component, *args)
+      if component.respond_to?(:call)
+        component.call(*args)
+      elsif component.respond_to?(:parse)
+        component.parse(*args)
+      elsif component.respond_to?(:decode)
+        component.decode(*args)
+      else
+        raise ReplicationError, "#{component.class} must respond to #call, #parse, or #decode"
+      end
+    end
+
+    def effective_runner
+      @runner ||= build_runner
+    end
+
+    def effective_parser
+      @parser ||= build_parser
+    end
+
+    def effective_decoder
+      @decoder ||= build_decoder
+    end
+
+    def effective_source_adapter
+      @source_adapter ||= build_source_adapter
+    end
+
+    def build_runner
+      require_optional!("pgoutput_client", "pgoutput-client")
+      Pgoutput::Client::Runner.new(
+        database_url: database_url,
+        slot_name: config.dig("replication", "slot"),
+        publication_names: [config.dig("replication", "publication")],
+        start_lsn: config.dig("replication", "start_lsn"),
+        auto_create_slot: config.dig("replication", "auto_create_slot") || false
+      )
+    end
+
+    def build_parser
+      require_any!(["pgoutput_parser", "pgoutput/parser"], "pgoutput-parser")
+      constant_or_nil("Pgoutput::Parser") || constant_or_nil("Pgoutput::Parser::Parser")
+    end
+
+    def build_decoder
+      require_any!(["pgoutput_decoder", "pgoutput/decoder"], "pgoutput-decoder")
+      constant_or_nil("Pgoutput::Decoder") || constant_or_nil("Pgoutput::Decoder::ValueDecoder")
+    end
+
+    def build_source_adapter
+      require_optional!("cdc_core", "cdc-core")
+      require_any!(["pgoutput_source_adapter", "pgoutput/source_adapter/cdc"], "pgoutput-source-adapter")
+
+      adapter_class = constant_or_nil("Pgoutput::SourceAdapter::Cdc")
+      raise ReplicationError, "Pgoutput::SourceAdapter::Cdc is unavailable" unless adapter_class
+
+      adapter_class.new
+    end
+
+    def database_url
+      password = ENV.fetch(config.dig("postgres", "password_env"), "")
+      user = config.dig("postgres", "username")
+      host = config.dig("postgres", "host")
+      port = config.dig("postgres", "port")
+      database = config.dig("postgres", "database")
+      "postgres://#{user}:#{password}@#{host}:#{port}/#{database}"
+    end
+
+    def require_optional!(feature, gem_name)
+      require feature
+    rescue LoadError => e
+      raise ReplicationError, "#{gem_name} is required for live pgoutput replication: #{e.message}"
+    end
+
+    def require_any!(features, gem_name)
+      errors = []
+      features.each do |feature|
+        require feature
+        return true
+      rescue LoadError => e
+        errors << e.message
+      end
+      raise ReplicationError, "#{gem_name} is required for live pgoutput replication: #{errors.join("; ")}"
+    end
+
+    def constant_or_nil(name)
+      name.split("::").reduce(Object) { |scope, const_name| scope.const_get(const_name, false) }
+    rescue NameError
+      nil
+    end
+  end
+end

data/lib/mammoth/replication_consumer.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Consumes CDC-core work items from Mammoth's configured replication source.
+  #
+  # ReplicationConsumer is the boundary between upstream CDC ingestion and
+  # sink delivery. Live PostgreSQL ingestion is delegated to {PgoutputSource};
+  # injected sources remain available for unit tests, demos, and e2e fixtures.
+  class ReplicationConsumer
+    attr_reader :config, :source, :adapter
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param source [#each, nil] injectable CDC work stream
+    # @param adapter [#call, nil] optional adapter for injected raw events
+    def initialize(config, source: nil, adapter: nil)
+      @config = config
+      @source = source
+      @adapter = adapter
+    end
+
+    # Return the configured replication slot.
+    #
+    # @return [String]
+    def slot
+      config.dig("replication", "slot")
+    end
+
+    # Return the configured publication.
+    #
+    # @return [String]
+    def publication
+      config.dig("replication", "publication")
+    end
+
+    # Consume normalized CDC work from the configured source.
+    #
+    # @yieldparam event [Object] CDC::Core::ChangeEvent-compatible event
+    # @return [Integer] number of consumed events
+    def start
+      return enum_for(:start) unless block_given?
+
+      count = 0
+      each_event do |event|
+        yield event
+        count += 1
+      end
+      count
+    end
+
+    private
+
+    def each_event
+      effective_source.each do |raw_work|
+        normalize(raw_work).each { |event| yield event }
+      end
+    end
+
+    def effective_source
+      source || PgoutputSource.new(config)
+    end
+
+    def normalize(raw_work)
+      adapted = adapter ? adapter.call(raw_work) : raw_work
+      flatten_cdc_work(adapted)
+    end
+
+    def flatten_cdc_work(work)
+      if transaction_envelope?(work)
+        work.events
+      else
+        Array(work).flat_map { |item| transaction_envelope?(item) ? item.events : item }
+      end
+    end
+
+    def transaction_envelope?(work)
+      work.respond_to?(:events) && work.respond_to?(:transaction_id)
+    end
+  end
+end

data/lib/mammoth/sql/__bootstrap__.sql

@@ -0,0 +1,55 @@
+CREATE TABLE IF NOT EXISTS schema_migrations (
+  version TEXT PRIMARY KEY,
+  applied_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS checkpoints (
+  id INTEGER PRIMARY KEY,
+  source_name TEXT NOT NULL,
+  slot_name TEXT NOT NULL,
+  publication_name TEXT NOT NULL,
+  last_lsn TEXT,
+  updated_at TEXT NOT NULL,
+
+  UNIQUE (source_name, slot_name)
+);
+
+CREATE TABLE IF NOT EXISTS dead_letters (
+  id INTEGER PRIMARY KEY,
+  event_id TEXT NOT NULL,
+  source_name TEXT NOT NULL,
+  destination_name TEXT NOT NULL,
+  operation TEXT NOT NULL,
+
+  namespace TEXT,
+  entity TEXT,
+  source_position TEXT,
+
+  payload_json TEXT NOT NULL,
+
+  error_class TEXT,
+  error_message TEXT,
+  retry_count INTEGER NOT NULL DEFAULT 0,
+
+  status TEXT NOT NULL DEFAULT 'pending',
+  failed_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+
+  CHECK (status IN ('pending', 'resolved', 'ignored')),
+  CHECK (retry_count >= 0)
+);
+
+CREATE INDEX IF NOT EXISTS idx_dead_letters_status
+ON dead_letters(status);
+
+CREATE INDEX IF NOT EXISTS idx_dead_letters_destination
+ON dead_letters(destination_name);
+
+CREATE INDEX IF NOT EXISTS idx_dead_letters_source_position
+ON dead_letters(source_position);
+
+CREATE INDEX IF NOT EXISTS idx_dead_letters_entity
+ON dead_letters(namespace, entity);
+
+CREATE INDEX IF NOT EXISTS idx_dead_letters_failed_at
+ON dead_letters(failed_at);

data/lib/mammoth/sqlite_store.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "sqlite3"
+require "time"
+
+module Mammoth
+  # Owns Mammoth's local SQLite operational database.
+  #
+  # The SQLite database is Mammoth's operational memory. It stores only boring,
+  # inspectable state required for reliability: schema versions, checkpoints,
+  # and dead letters.
+  class SQLiteStore
+    DEFAULT_DB_PATH = File.expand_path("../../.sqlite3/mammoth.db", __dir__)
+    MIGRATION_DIR = File.expand_path("sql", __dir__)
+    BOOTSTRAP_FILE = "__bootstrap__.sql"
+    BOOTSTRAP_VERSION = "__bootstrap__"
+    MIGRATIONS_TABLE = "schema_migrations"
+
+    attr_reader :path
+
+    # Open and return a connected store.
+    #
+    # @param path [String] SQLite database path
+    # @return [Mammoth::SQLiteStore]
+    def self.connect(path = DEFAULT_DB_PATH)
+      new(path).connect
+    end
+
+    # @param path [String] SQLite database path
+    def initialize(path = DEFAULT_DB_PATH)
+      @path = path
+      @database = nil
+    end
+
+    # Open the SQLite database and configure conservative operational pragmas.
+    #
+    # @return [Mammoth::SQLiteStore] self
+    def connect
+      FileUtils.mkdir_p(File.dirname(path))
+      @database = SQLite3::Database.new(path)
+      @database.results_as_hash = true
+      execute_pragma("journal_mode", "WAL")
+      execute_pragma("foreign_keys", "ON")
+      execute_pragma("busy_timeout", 5000)
+      self
+    rescue SQLite3::Exception => e
+      raise StoreError, "failed to open SQLite database #{path}: #{e.message}"
+    end
+
+    # @return [SQLite3::Database] connected database
+    def database
+      @database || connect.database
+    end
+
+    # Apply the initial schema if it has not yet been applied.
+    #
+    # @return [Mammoth::SQLiteStore] self
+    def bootstrap!
+      return self if bootstrapped?
+
+      apply_migration!(BOOTSTRAP_FILE, BOOTSTRAP_VERSION)
+      self
+    end
+
+    # Apply a migration file unless its version already exists.
+    #
+    # @param sql_file [String] SQL file name under lib/mammoth/sql
+    # @return [Mammoth::SQLiteStore] self
+    def migrate!(sql_file)
+      bootstrap!
+      version_name = File.basename(sql_file, ".sql")
+      return self if version_exists?(version_name)
+
+      apply_migration!(sql_file, version_name)
+      self
+    end
+
+    # @return [Boolean] true when the initial schema is already applied
+    def bootstrapped?
+      table_exists?(MIGRATIONS_TABLE) && version_exists?(BOOTSTRAP_VERSION)
+    end
+
+    # @param table_name [String] table name
+    # @return [Boolean] true when a table exists
+    def table_exists?(table_name)
+      !database.execute(
+        "SELECT 1 FROM sqlite_schema WHERE type = ? AND name = ? LIMIT 1",
+        ["table", table_name]
+      ).empty?
+    end
+
+    # @param version_name [String] migration version
+    # @return [Boolean] true when a migration version exists
+    def version_exists?(version_name)
+      return false unless table_exists?(MIGRATIONS_TABLE)
+
+      !database.execute(
+        "SELECT 1 FROM schema_migrations WHERE version = ? LIMIT 1",
+        [version_name]
+      ).empty?
+    end
+
+    # Return table names in the operational database.
+    #
+    # @return [Array<String>] table names
+    def tables
+      database.execute(
+        "SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name"
+      ).map { |row| row.fetch("name") }
+    end
+
+    private
+
+    def apply_migration!(sql_file, version_name)
+      sql_path = File.expand_path(sql_file, MIGRATION_DIR)
+      raise StoreError, "migration file not found: #{sql_path}" unless File.file?(sql_path)
+
+      database.transaction do
+        database.execute_batch(File.read(sql_path))
+        database.execute(
+          "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)",
+          [version_name, Time.now.utc.iso8601]
+        )
+      end
+    rescue SQLite3::Exception => e
+      raise StoreError, "failed to apply migration #{sql_file}: #{e.message}"
+    end
+
+    def execute_pragma(name, value)
+      database.execute("PRAGMA #{name} = #{value}")
+    end
+  end
+end

data/lib/mammoth/status.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Builds and prints a boring operational status snapshot.
+  class Status
+    attr_reader :config, :sqlite_store
+
+    # Print status for a configuration and optional SQLite store.
+    #
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param sqlite_store [Mammoth::SQLiteStore, nil] operational store
+    # @return [void]
+    def self.call(config, sqlite_store: nil)
+      new(config, sqlite_store: sqlite_store).call
+    end
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param sqlite_store [Mammoth::SQLiteStore, nil] operational store
+    def initialize(config, sqlite_store: nil)
+      @config = config
+      @sqlite_store = sqlite_store
+    end
+
+    # Print the status snapshot.
+    #
+    # @return [void]
+    def call
+      puts "Mammoth: #{config.dig("mammoth", "name")}"
+      puts "Runtime: not started"
+      puts "SQLite: #{sqlite_path}"
+      puts "Webhook: #{config.dig("webhook", "name")}"
+      print_store_state if sqlite_store
+    end
+
+    private
+
+    def sqlite_path
+      config.dig("sqlite", "path")
+    end
+
+    def print_store_state
+      store = sqlite_store.bootstrap!
+      puts "Tables: #{store.tables.join(", ")}"
+      puts "Checkpoints: #{CheckpointStore.new(store).count}"
+      puts "Dead Letters: #{DeadLetterStore.new(store).count}"
+    end
+  end
+end

data/lib/mammoth/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mammoth
+  VERSION = "0.0.0"
+end

data/lib/mammoth/webhook_sink.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module Mammoth
+  # Delivers normalized Mammoth events to a webhook endpoint.
+  class WebhookSink
+    SUCCESS_RANGE = 200..299
+
+    attr_reader :name, :url, :timeout_seconds
+
+    # @param name [String] destination name
+    # @param url [String] webhook endpoint URL
+    # @param timeout_seconds [Integer] HTTP open/read timeout in seconds
+    def initialize(name:, url:, timeout_seconds: 5)
+      @name = name
+      @url = URI(url)
+      @timeout_seconds = timeout_seconds
+    end
+
+    # Build a sink from Mammoth configuration.
+    #
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @return [Mammoth::WebhookSink]
+    def self.from_config(config)
+      new(
+        name: config.dig("webhook", "name"),
+        url: config.dig("webhook", "url"),
+        timeout_seconds: config.dig("webhook", "timeout_seconds")
+      )
+    end
+
+    # Deliver an event to the webhook endpoint.
+    #
+    # @param event [Hash, #to_h] normalized event
+    # @return [Hash] delivery result
+    # @raise [Mammoth::DeliveryError] when delivery fails
+    def deliver(event)
+      payload = EventSerializer.call(event)
+      response = perform_request(payload)
+      return delivery_result(payload, response) if SUCCESS_RANGE.cover?(response.code.to_i)
+
+      raise DeliveryError, "webhook #{name} returned HTTP #{response.code}"
+    rescue Timeout::Error, SystemCallError, SocketError, JSON::GeneratorError => e
+      raise DeliveryError, "webhook #{name} delivery failed: #{e.message}"
+    end
+
+    private
+
+    def perform_request(payload)
+      Net::HTTP.start(url.host, url.port, use_ssl: url.scheme == "https", open_timeout: timeout_seconds,
+                                          read_timeout: timeout_seconds) do |http|
+        http.request(build_request(payload))
+      end
+    end
+
+    def build_request(payload)
+      Net::HTTP::Post.new(url.request_uri).tap do |request|
+        request["Content-Type"] = "application/json"
+        request.body = JSON.generate(payload)
+      end
+    end
+
+    def delivery_result(payload, response)
+      {
+        event_id: payload.fetch("event_id"),
+        destination: name,
+        status: "delivered",
+        http_status: response.code.to_i
+      }
+    end
+  end
+end

data/lib/mammoth.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "mammoth/version"
+require_relative "mammoth/errors"
+require_relative "mammoth/configuration"
+require_relative "mammoth/status"
+require_relative "mammoth/sqlite_store"
+require_relative "mammoth/checkpoint_store"
+require_relative "mammoth/dead_letter_store"
+require_relative "mammoth/event_serializer"
+require_relative "mammoth/webhook_sink"
+require_relative "mammoth/delivery_worker"
+require_relative "mammoth/pgoutput_source"
+require_relative "mammoth/replication_consumer"
+require_relative "mammoth/application"
+require_relative "mammoth/cli"
+
+# Mammoth is a self-hosted PostgreSQL event relay.
+#
+# Mammoth v0.1.0 focuses on a deliberately small, boring product slice:
+# PostgreSQL change events are normalized, persisted through local operational
+# state, and delivered to webhook destinations.
+module Mammoth
+end