mammoth 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19aba46c6ef2ae1099287e37281015b747dde1b32c94cbd1c8ec8a23d6b1a19b
-  data.tar.gz: 21430175d42c71e83eaede3f70390a87d760a9ac2d8b9b4dedcfb446feefaaa0
+  metadata.gz: 5d7d570675d2b6731d783c6eba0f347ab00c1d7f9e524c20eef4908af129c41a
+  data.tar.gz: f14c3223b20cd1ea8e7d795e1cd2fe4e512766a0a4ed275515e4d1174bb3d6f5
 SHA512:
-  metadata.gz: b29ea1b4ace38ffa6eb155251dbcd0056e400849aa6a37093d4ecec09cc314089c36523b22b3bbbf898b0247fb6918e3320ef0f14b24d280fed91df2a2287f9a
-  data.tar.gz: b2e6f58f77884641970e838a5aaa8c2c1ef0a2531b868a53c3ae8ee450387128b04b39c2661e5fac507c82d05b11739a913285c8f9c2052db137810881073a3e
+  metadata.gz: cf649aba9068bdc1c3b0e01c1254f8f16804e80ec88de725d2743e99be9b9010bd880590fde044c9d7dff3b0b99fac54c6c9776af4504fb566eeceb1e5b00fd9
+  data.tar.gz: a7396f91f4fe84b3780f9b176931d728707b6e55ecab9b168b5eb3e87cb384c2066d9ed0b944dd14405d2e06a02e0e363a28348325578bb652aa4ba1df56caee

data/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+## 0.3.0
+
+### Added
+
+- Added dead-letter inspection and replay commands for operational recovery.
+- Added dead-letter support for list, show, and replay, with replay routing both event and transaction dead letters back through the existing delivery path and resolving successful rows
+
+
 ## 0.2.0
 
 ### Added

data/README.md

@@ -47,6 +47,7 @@ https://kanutocd.github.io/mammoth/Mammoth.html
 - dead letter persistence
 - webhook delivery sink
 - delivery worker with retry, checkpoint, and DLQ handling
+- dead-letter inspection and replay commands
 - CDC-core event serialization boundary
 - CDC Ecosystem source-adapter integration boundary
 - Docker image support

data/lib/mammoth/cli.rb

@@ -5,6 +5,16 @@ require "json"
 module Mammoth
   # Small command dispatcher for Mammoth's operator-facing CLI.
   class CLI
+    # Internal replay envelope used for transaction dead-letter recovery.
+    DEAD_LETTER_TRANSACTION_ENVELOPE = Data.define(
+      :events,
+      :transaction_id,
+      :source_position,
+      :commit_lsn,
+      :committed_at,
+      :metadata
+    )
+
     # Human-readable command usage printed for invalid or incomplete invocations.
     USAGE = [
       "Usage:",
@@ -13,7 +23,10 @@ module Mammoth
       "  mammoth bootstrap CONFIG",
       "  mammoth status CONFIG",
       "  mammoth start CONFIG",
-      "  mammoth deliver-sample CONFIG EVENT_JSON"
+      "  mammoth deliver-sample CONFIG EVENT_JSON",
+      "  mammoth dead-letters list CONFIG [--status STATUS] [--limit N]",
+      "  mammoth dead-letters show CONFIG ID",
+      "  mammoth dead-letters replay CONFIG [ID ...]"
     ].join("\n")
 
     # Run the CLI.
@@ -42,6 +55,7 @@ module Mammoth
       when "status" then status
       when "start" then start
       when "deliver-sample" then deliver_sample
+      when "dead-letters" then dead_letters
       else
         warn USAGE
         1
@@ -109,6 +123,13 @@ module Mammoth
       raise ConfigurationError, "invalid event JSON in #{event_path}: #{e.message}"
     end
 
+    # Dispatch the nested dead-letter command group.
+    #
+    # @return [Integer] process status code
+    def dead_letters
+      DeadLetterCommands.call(argv)
+    end
+
     def load_config
       raise ConfigurationError, "configuration path required\n#{USAGE}" unless config_path
 

data/lib/mammoth/dead_letter_commands.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Mammoth
+  # Operator commands for inspecting and replaying dead letters.
+  # rubocop:disable Metrics/ClassLength
+  class DeadLetterCommands
+    # Internal replay envelope used for transaction dead-letter recovery.
+    DEAD_LETTER_TRANSACTION_ENVELOPE = Data.define(
+      :events,
+      :transaction_id,
+      :source_position,
+      :commit_lsn,
+      :committed_at,
+      :metadata
+    )
+
+    attr_reader :argv
+
+    # @param argv [Array<String>] command line arguments
+    def self.call(argv)
+      new(argv).call
+    end
+
+    # @param argv [Array<String>] command line arguments
+    def initialize(argv)
+      @argv = argv
+    end
+
+    # Dispatch the nested dead-letter subcommand.
+    #
+    # @return [Integer] process status code
+    def call
+      case command
+      when "list" then list
+      when "show" then show
+      when "replay" then replay
+      else
+        raise ConfigurationError, "dead-letters subcommand required\n#{CLI::USAGE}"
+      end
+    rescue Mammoth::Error => e
+      warn e.message
+      1
+    end
+
+    private
+
+    def command
+      argv.fetch(1, nil)
+    end
+
+    def config_path
+      argv.fetch(2, nil)
+    end
+
+    def list
+      rows = dead_letter_store.rows(status: list_options.fetch(:status), limit: list_options.fetch(:limit))
+      puts list_header
+      rows.each { |row| puts list_row(row) }
+      0
+    end
+
+    def show
+      row = fetch_dead_letter!(dead_letter_id)
+      puts JSON.pretty_generate(show_payload(row))
+      0
+    end
+
+    def replay
+      rows = replay_rows
+      raise ConfigurationError, "no dead letters found to replay" if rows.empty?
+
+      rows.each do |row|
+        result = replay_row(row)
+        dead_letter_store.resolve(row.fetch("id")) unless result.fetch(:status) == "dead_lettered"
+        puts replay_message(row, result)
+      end
+      0
+    end
+
+    def load_config
+      raise ConfigurationError, "configuration path required\n#{CLI::USAGE}" unless config_path
+
+      Configuration.load(config_path)
+    end
+
+    def dead_letter_store
+      @dead_letter_store ||= DeadLetterStore.new(SQLiteStore.connect(load_config.dig("sqlite", "path")).bootstrap!)
+    end
+
+    def worker
+      @worker ||= Application.new(load_config).delivery_worker
+    end
+
+    def dead_letter_id
+      raw_id = argv.fetch(3, nil)
+      raise ConfigurationError, "dead letter id required\n#{CLI::USAGE}" unless raw_id
+
+      Integer(raw_id)
+    rescue ArgumentError
+      raise ConfigurationError, "dead letter id must be an integer"
+    end
+
+    # Parse dead-letter list filters and pagination.
+    #
+    # @return [Hash] list options
+    def list_options
+      options = { status: "pending", limit: 100 }
+      index = 0
+      args = argv.drop(3)
+
+      # rubocop:disable Style/WhileUntilModifier
+      while index < args.length
+        index = parse_list_option(args, index, options)
+      end
+      # rubocop:enable Style/WhileUntilModifier
+
+      options
+    end
+
+    def parse_list_option(args, index, options)
+      case args.fetch(index)
+      when "--status"
+        options[:status] = args.fetch(index + 1)
+        index + 2
+      when "--limit"
+        options[:limit] = Integer(args.fetch(index + 1))
+        index + 2
+      else
+        raise ConfigurationError, "unknown option #{args[index]}\n#{CLI::USAGE}" if args[index].start_with?("--")
+
+        raise ConfigurationError, "unexpected argument #{args[index]}\n#{CLI::USAGE}"
+      end
+    rescue ArgumentError, IndexError
+      raise ConfigurationError, "dead letter limit must be an integer"
+    end
+
+    # Resolve the rows that should be replayed.
+    #
+    # @return [Array<Hash>] replay rows
+    def replay_rows
+      ids = argv.drop(3)
+      return dead_letter_store.pending if ids.empty?
+
+      ids.map do |raw_id|
+        id = Integer(raw_id)
+        row = dead_letter_store.fetch(id)
+        raise ConfigurationError, "dead letter not found: #{id}" unless row
+
+        row
+      rescue ArgumentError
+        raise ConfigurationError, "dead letter id must be an integer"
+      end
+    end
+
+    def fetch_dead_letter!(id)
+      row = dead_letter_store.fetch(id)
+      raise ConfigurationError, "dead letter not found: #{id}" unless row
+
+      row
+    end
+
+    def replay_row(row)
+      payload = JSON.parse(row.fetch("payload_json"))
+      transaction_payload?(payload) ? worker.deliver_transaction(transaction_envelope(payload)) : worker.deliver(payload)
+    end
+
+    def transaction_payload?(payload)
+      payload.fetch("type", nil) == TransactionEnvelopeSerializer::PAYLOAD_TYPE
+    end
+
+    def transaction_envelope(payload)
+      DEAD_LETTER_TRANSACTION_ENVELOPE.new(
+        payload.fetch("events"),
+        payload.fetch("transaction_id"),
+        payload["source_position"],
+        payload["commit_lsn"],
+        payload["committed_at"],
+        payload["metadata"] || {}
+      )
+    end
+
+    def show_payload(row)
+      row.merge("payload" => JSON.parse(row.fetch("payload_json")))
+    end
+
+    def list_header
+      format(
+        "%<id>-4s  %<status>-10s  %<destination>-18s  %<retries>-8s  %<failed_at>-19s  %<event>s",
+        id: "ID",
+        status: "STATUS",
+        destination: "DESTINATION",
+        retries: "RETRIES",
+        failed_at: "FAILED_AT",
+        event: "EVENT"
+      )
+    end
+
+    def list_row(row)
+      format(
+        "%<id>-4s  %<status>-10s  %<destination>-18s  %<retries>-8s  %<failed_at>-19s  %<event>s",
+        id: row.fetch("id"),
+        status: row.fetch("status"),
+        destination: row.fetch("destination_name"),
+        retries: row.fetch("retry_count"),
+        failed_at: row.fetch("failed_at"),
+        event: row.fetch("event_id")
+      )
+    end
+
+    def replay_message(row, result)
+      "Dead letter #{row.fetch("id")}: #{result.fetch(:status)}"
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/mammoth/dead_letter_store.rb

@@ -66,10 +66,31 @@ module Mammoth
     # @param limit [Integer] maximum number of rows
     # @return [Array<Hash>] pending dead letter rows
     def pending(limit: 100)
-      database.execute(
-        "SELECT * FROM dead_letters WHERE status = ? ORDER BY failed_at ASC LIMIT ?",
-        ["pending", limit]
-      )
+      rows(status: "pending", limit: limit)
+    end
+
+    # Fetch dead letters, optionally filtered by status.
+    #
+    # @param status [String, nil] optional dead-letter status filter
+    # @param limit [Integer] maximum number of rows
+    # @return [Array<Hash>] dead letter rows
+    def rows(status: nil, limit: 100)
+      if status && status != "all"
+        return database.execute(
+          "SELECT * FROM dead_letters WHERE status = ? ORDER BY failed_at ASC LIMIT ?",
+          [status, limit]
+        )
+      end
+
+      database.execute("SELECT * FROM dead_letters ORDER BY failed_at ASC LIMIT ?", [limit])
+    end
+
+    # Fetch one dead letter by id.
+    #
+    # @param id [Integer] dead letter id
+    # @return [Hash, nil] dead letter row
+    def fetch(id)
+      database.get_first_row("SELECT * FROM dead_letters WHERE id = ?", [id])
     end
 
     # Count dead letters by status.

data/lib/mammoth/delivered_envelope_store.rb

@@ -9,6 +9,7 @@ module Mammoth
   # keeps a small delivery ledger so a transaction replayed by the upstream
   # replication source after restart does not have to be delivered downstream again.
   class DeliveredEnvelopeStore
+    # SQLite schema used to bootstrap the delivered-envelope ledger.
     SCHEMA = <<~SQL
       CREATE TABLE IF NOT EXISTS delivered_envelopes (
         id INTEGER PRIMARY KEY,
@@ -81,6 +82,9 @@ module Mammoth
       database.execute("SELECT * FROM delivered_envelopes ORDER BY id")
     end
 
+    # Count delivered envelopes.
+    #
+    # @return [Integer] delivered envelope count
     def count
       database.get_first_value("SELECT COUNT(*) FROM delivered_envelopes")
     end

data/lib/mammoth/delivery_processor.rb

@@ -56,6 +56,10 @@ module Mammoth
       process(work)
     end
 
+    # Process one work item using the configured delivery unit.
+    #
+    # @param work [Object] event or transaction envelope
+    # @return [Hash] delivery summary
     def process(work)
       case delivery_unit
       when :transaction

data/lib/mammoth/sources/postgres.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Mammoth
+  # Namespace for Mammoth source adapters.
   module Sources
     # Concrete PostgreSQL CDC source for Mammoth.
     #

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/mammoth/webhook_sink.rb

@@ -13,7 +13,9 @@ module Mammoth
     # HTTP status range treated as successful webhook delivery.
     SUCCESS_RANGE = 200..299
 
+    # Supported webhook signing algorithm.
     SIGNING_ALGORITHM = "hmac_sha256"
+    # Prefix added to generated webhook signatures.
     SIGNATURE_PREFIX = "sha256="
 
     attr_reader :name, :url, :timeout_seconds, :headers, :signing

data/lib/mammoth.rb

@@ -18,6 +18,7 @@ require_relative "mammoth/sources/postgres"
 require_relative "mammoth/cdc_source"
 require_relative "mammoth/replication_consumer"
 require_relative "mammoth/application"
+require_relative "mammoth/dead_letter_commands"
 require_relative "mammoth/cli"
 
 # Mammoth is a self-hosted PostgreSQL event relay.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mammoth
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -150,6 +150,7 @@ files:
 - lib/mammoth/cli.rb
 - lib/mammoth/concurrent_delivery_runtime.rb
 - lib/mammoth/configuration.rb
+- lib/mammoth/dead_letter_commands.rb
 - lib/mammoth/dead_letter_store.rb
 - lib/mammoth/delivered_envelope_store.rb
 - lib/mammoth/delivery_processor.rb