mammoth 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19aba46c6ef2ae1099287e37281015b747dde1b32c94cbd1c8ec8a23d6b1a19b
-  data.tar.gz: 21430175d42c71e83eaede3f70390a87d760a9ac2d8b9b4dedcfb446feefaaa0
+  metadata.gz: 8fdbb7afb90f584e11024c75d7300f54978f5e5b1a3628597bb460f96a306c22
+  data.tar.gz: 9ced2250d7e2f95558a8bcce51809aa4d670d49953d8d970e168a0c48cf90215
 SHA512:
-  metadata.gz: b29ea1b4ace38ffa6eb155251dbcd0056e400849aa6a37093d4ecec09cc314089c36523b22b3bbbf898b0247fb6918e3320ef0f14b24d280fed91df2a2287f9a
-  data.tar.gz: b2e6f58f77884641970e838a5aaa8c2c1ef0a2531b868a53c3ae8ee450387128b04b39c2661e5fac507c82d05b11739a913285c8f9c2052db137810881073a3e
+  metadata.gz: cd6b5b3246cf70adb9beb0cc9243e6a35788d20f3442ab406bca66a4cb08bd4a7d80639386c4e42291b518190a0198f960ed9e16cbd165f335a7f3fee10317b7
+  data.tar.gz: 87aa4ee77c38aa69d1a505bf7eb2b838de86504316840dfc26b4e875870333f64da906c85aa180a205e5e3bff4f439e36d2f82ffa79d650117d9f5ccc4c29b69

data/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+## 0.4.0
+
+### Added
+
+- Added optional observability support with `/healthz`, `/readyz`, and `/metrics` endpoints, plus CLI startup, configuration, and documentation coverage.
+
+## 0.4.0
+
+### Added
+
+- Added dead-letter inspection and replay commands for operational recovery.
+- Added dead-letter support for list, show, and replay, with replay routing both event and transaction dead letters back through the existing delivery path and resolving successful rows
+
+
 ## 0.2.0
 
 ### Added

data/README.md

@@ -47,11 +47,13 @@ https://kanutocd.github.io/mammoth/Mammoth.html
 - dead letter persistence
 - webhook delivery sink
 - delivery worker with retry, checkpoint, and DLQ handling
+- dead-letter inspection and replay commands
 - CDC-core event serialization boundary
 - CDC Ecosystem source-adapter integration boundary
 - Docker image support
 - public Helm chart support
 - unit and e2e test tasks
+- Health and metrics endpoints
 
 ## Boundary
 
@@ -83,6 +85,7 @@ bundle exec ./exe/mammoth validate config/mammoth.example.yml
 bundle exec ./exe/mammoth bootstrap config/mammoth.example.yml
 bundle exec ./exe/mammoth status config/mammoth.example.yml
 bundle exec ./exe/mammoth start config/mammoth.example.yml
+bundle exec ./exe/mammoth observability config/mammoth.example.yml
 ```
 
 Deliver a single normalized event JSON file through Mammoth's delivery path:

data/config/mammoth.example.yml

@@ -274,6 +274,13 @@ sqlite:
   # In Kubernetes, back this path with a PersistentVolumeClaim.
   path: data/mammoth.db
 
+observability:
+  # Bind host for the optional health, readiness, and metrics server.
+  host: 0.0.0.0
+
+  # HTTP port for /healthz, /readyz, and /metrics.
+  port: 9393
+
 logging:
   # Log verbosity.
   #

data/config/mammoth.schema.json

@@ -319,6 +319,24 @@
         }
       },
       "description": "Downstream delivery runtime settings. Runtime concurrency affects webhook/destination delivery only, not PostgreSQL replication stream count."
+    },
+    "observability": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "host": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Bind host for the optional Mammoth observability HTTP server exposing /healthz, /readyz, and /metrics."
+        },
+        "port": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 65535,
+          "description": "TCP port for the optional Mammoth observability HTTP server."
+        }
+      },
+      "description": "Optional health, readiness, and metrics endpoint settings."
     }
   },
   "description": "Mammoth relay configuration. Mammoth consumes PostgreSQL logical replication changes and delivers them to downstream destinations while preserving checkpoint, retry, and dead-letter semantics."

data/lib/mammoth/cli.rb

@@ -5,6 +5,16 @@ require "json"
 module Mammoth
   # Small command dispatcher for Mammoth's operator-facing CLI.
   class CLI
+    # Internal replay envelope used for transaction dead-letter recovery.
+    DEAD_LETTER_TRANSACTION_ENVELOPE = Data.define(
+      :events,
+      :transaction_id,
+      :source_position,
+      :commit_lsn,
+      :committed_at,
+      :metadata
+    )
+
     # Human-readable command usage printed for invalid or incomplete invocations.
     USAGE = [
       "Usage:",
@@ -13,7 +23,11 @@ module Mammoth
       "  mammoth bootstrap CONFIG",
       "  mammoth status CONFIG",
       "  mammoth start CONFIG",
-      "  mammoth deliver-sample CONFIG EVENT_JSON"
+      "  mammoth deliver-sample CONFIG EVENT_JSON",
+      "  mammoth dead-letters list CONFIG [--status STATUS] [--limit N]",
+      "  mammoth dead-letters show CONFIG ID",
+      "  mammoth dead-letters replay CONFIG [ID ...]",
+      "  mammoth observability CONFIG"
     ].join("\n")
 
     # Run the CLI.
@@ -42,6 +56,8 @@ module Mammoth
       when "status" then status
       when "start" then start
       when "deliver-sample" then deliver_sample
+      when "dead-letters" then dead_letters
+      when "observability" then observability
       else
         warn USAGE
         1
@@ -109,6 +125,21 @@ module Mammoth
       raise ConfigurationError, "invalid event JSON in #{event_path}: #{e.message}"
     end
 
+    # Dispatch the nested dead-letter command group.
+    #
+    # @return [Integer] process status code
+    def dead_letters
+      DeadLetterCommands.call(argv)
+    end
+
+    def observability
+      config = load_config
+      server = ObservabilityServer.new(config)
+      puts "Mammoth observability listening on #{server.host}:#{server.port}"
+      server.start
+      0
+    end
+
     def load_config
       raise ConfigurationError, "configuration path required\n#{USAGE}" unless config_path
 

data/lib/mammoth/dead_letter_commands.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Mammoth
+  # Operator commands for inspecting and replaying dead letters.
+  # rubocop:disable Metrics/ClassLength
+  class DeadLetterCommands
+    # Internal replay envelope used for transaction dead-letter recovery.
+    DEAD_LETTER_TRANSACTION_ENVELOPE = Data.define(
+      :events,
+      :transaction_id,
+      :source_position,
+      :commit_lsn,
+      :committed_at,
+      :metadata
+    )
+
+    attr_reader :argv
+
+    # @param argv [Array<String>] command line arguments
+    def self.call(argv)
+      new(argv).call
+    end
+
+    # @param argv [Array<String>] command line arguments
+    def initialize(argv)
+      @argv = argv
+    end
+
+    # Dispatch the nested dead-letter subcommand.
+    #
+    # @return [Integer] process status code
+    def call
+      case command
+      when "list" then list
+      when "show" then show
+      when "replay" then replay
+      else
+        raise ConfigurationError, "dead-letters subcommand required\n#{CLI::USAGE}"
+      end
+    rescue Mammoth::Error => e
+      warn e.message
+      1
+    end
+
+    private
+
+    def command
+      argv.fetch(1, nil)
+    end
+
+    def config_path
+      argv.fetch(2, nil)
+    end
+
+    def list
+      rows = dead_letter_store.rows(status: list_options.fetch(:status), limit: list_options.fetch(:limit))
+      puts list_header
+      rows.each { |row| puts list_row(row) }
+      0
+    end
+
+    def show
+      row = fetch_dead_letter!(dead_letter_id)
+      puts JSON.pretty_generate(show_payload(row))
+      0
+    end
+
+    def replay
+      rows = replay_rows
+      raise ConfigurationError, "no dead letters found to replay" if rows.empty?
+
+      rows.each do |row|
+        result = replay_row(row)
+        dead_letter_store.resolve(row.fetch("id")) unless result.fetch(:status) == "dead_lettered"
+        puts replay_message(row, result)
+      end
+      0
+    end
+
+    def load_config
+      raise ConfigurationError, "configuration path required\n#{CLI::USAGE}" unless config_path
+
+      Configuration.load(config_path)
+    end
+
+    def dead_letter_store
+      @dead_letter_store ||= DeadLetterStore.new(SQLiteStore.connect(load_config.dig("sqlite", "path")).bootstrap!)
+    end
+
+    def worker
+      @worker ||= Application.new(load_config).delivery_worker
+    end
+
+    def dead_letter_id
+      raw_id = argv.fetch(3, nil)
+      raise ConfigurationError, "dead letter id required\n#{CLI::USAGE}" unless raw_id
+
+      Integer(raw_id)
+    rescue ArgumentError
+      raise ConfigurationError, "dead letter id must be an integer"
+    end
+
+    # Parse dead-letter list filters and pagination.
+    #
+    # @return [Hash] list options
+    def list_options
+      options = { status: "pending", limit: 100 }
+      index = 0
+      args = argv.drop(3)
+
+      # rubocop:disable Style/WhileUntilModifier
+      while index < args.length
+        index = parse_list_option(args, index, options)
+      end
+      # rubocop:enable Style/WhileUntilModifier
+
+      options
+    end
+
+    def parse_list_option(args, index, options)
+      case args.fetch(index)
+      when "--status"
+        options[:status] = args.fetch(index + 1)
+        index + 2
+      when "--limit"
+        options[:limit] = Integer(args.fetch(index + 1))
+        index + 2
+      else
+        raise ConfigurationError, "unknown option #{args[index]}\n#{CLI::USAGE}" if args[index].start_with?("--")
+
+        raise ConfigurationError, "unexpected argument #{args[index]}\n#{CLI::USAGE}"
+      end
+    rescue ArgumentError, IndexError
+      raise ConfigurationError, "dead letter limit must be an integer"
+    end
+
+    # Resolve the rows that should be replayed.
+    #
+    # @return [Array<Hash>] replay rows
+    def replay_rows
+      ids = argv.drop(3)
+      return dead_letter_store.pending if ids.empty?
+
+      ids.map do |raw_id|
+        id = Integer(raw_id)
+        row = dead_letter_store.fetch(id)
+        raise ConfigurationError, "dead letter not found: #{id}" unless row
+
+        row
+      rescue ArgumentError
+        raise ConfigurationError, "dead letter id must be an integer"
+      end
+    end
+
+    def fetch_dead_letter!(id)
+      row = dead_letter_store.fetch(id)
+      raise ConfigurationError, "dead letter not found: #{id}" unless row
+
+      row
+    end
+
+    def replay_row(row)
+      payload = JSON.parse(row.fetch("payload_json"))
+      transaction_payload?(payload) ? worker.deliver_transaction(transaction_envelope(payload)) : worker.deliver(payload)
+    end
+
+    def transaction_payload?(payload)
+      payload.fetch("type", nil) == TransactionEnvelopeSerializer::PAYLOAD_TYPE
+    end
+
+    def transaction_envelope(payload)
+      DEAD_LETTER_TRANSACTION_ENVELOPE.new(
+        payload.fetch("events"),
+        payload.fetch("transaction_id"),
+        payload["source_position"],
+        payload["commit_lsn"],
+        payload["committed_at"],
+        payload["metadata"] || {}
+      )
+    end
+
+    def show_payload(row)
+      row.merge("payload" => JSON.parse(row.fetch("payload_json")))
+    end
+
+    def list_header
+      format(
+        "%<id>-4s  %<status>-10s  %<destination>-18s  %<retries>-8s  %<failed_at>-19s  %<event>s",
+        id: "ID",
+        status: "STATUS",
+        destination: "DESTINATION",
+        retries: "RETRIES",
+        failed_at: "FAILED_AT",
+        event: "EVENT"
+      )
+    end
+
+    def list_row(row)
+      format(
+        "%<id>-4s  %<status>-10s  %<destination>-18s  %<retries>-8s  %<failed_at>-19s  %<event>s",
+        id: row.fetch("id"),
+        status: row.fetch("status"),
+        destination: row.fetch("destination_name"),
+        retries: row.fetch("retry_count"),
+        failed_at: row.fetch("failed_at"),
+        event: row.fetch("event_id")
+      )
+    end
+
+    def replay_message(row, result)
+      "Dead letter #{row.fetch("id")}: #{result.fetch(:status)}"
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/mammoth/dead_letter_store.rb

@@ -66,10 +66,31 @@ module Mammoth
     # @param limit [Integer] maximum number of rows
     # @return [Array<Hash>] pending dead letter rows
     def pending(limit: 100)
-      database.execute(
-        "SELECT * FROM dead_letters WHERE status = ? ORDER BY failed_at ASC LIMIT ?",
-        ["pending", limit]
-      )
+      rows(status: "pending", limit: limit)
+    end
+
+    # Fetch dead letters, optionally filtered by status.
+    #
+    # @param status [String, nil] optional dead-letter status filter
+    # @param limit [Integer] maximum number of rows
+    # @return [Array<Hash>] dead letter rows
+    def rows(status: nil, limit: 100)
+      if status && status != "all"
+        return database.execute(
+          "SELECT * FROM dead_letters WHERE status = ? ORDER BY failed_at ASC LIMIT ?",
+          [status, limit]
+        )
+      end
+
+      database.execute("SELECT * FROM dead_letters ORDER BY failed_at ASC LIMIT ?", [limit])
+    end
+
+    # Fetch one dead letter by id.
+    #
+    # @param id [Integer] dead letter id
+    # @return [Hash, nil] dead letter row
+    def fetch(id)
+      database.get_first_row("SELECT * FROM dead_letters WHERE id = ?", [id])
     end
 
     # Count dead letters by status.

data/lib/mammoth/delivered_envelope_store.rb

@@ -9,6 +9,7 @@ module Mammoth
   # keeps a small delivery ledger so a transaction replayed by the upstream
   # replication source after restart does not have to be delivered downstream again.
   class DeliveredEnvelopeStore
+    # SQLite schema used to bootstrap the delivered-envelope ledger.
     SCHEMA = <<~SQL
       CREATE TABLE IF NOT EXISTS delivered_envelopes (
         id INTEGER PRIMARY KEY,
@@ -81,6 +82,9 @@ module Mammoth
       database.execute("SELECT * FROM delivered_envelopes ORDER BY id")
     end
 
+    # Count delivered envelopes.
+    #
+    # @return [Integer] delivered envelope count
     def count
       database.get_first_value("SELECT COUNT(*) FROM delivered_envelopes")
     end

data/lib/mammoth/delivery_processor.rb

@@ -56,6 +56,10 @@ module Mammoth
       process(work)
     end
 
+    # Process one work item using the configured delivery unit.
+    #
+    # @param work [Object] event or transaction envelope
+    # @return [Hash] delivery summary
     def process(work)
       case delivery_unit
       when :transaction

data/lib/mammoth/observability_server.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "json"
+require "webrick"
+
+module Mammoth
+  # Small HTTP server exposing Mammoth health, readiness, and metrics endpoints.
+  #
+  # The server is intentionally independent from the replication loop. Operators
+  # may run it as a sidecar-like process or in a separate process that points at
+  # the same SQLite operational database.
+  class ObservabilityServer
+    DEFAULT_HOST = "0.0.0.0"
+    DEFAULT_PORT = 9393
+
+    attr_reader :config, :host, :port, :sqlite_store, :logger, :server
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param host [String, nil] bind host override
+    # @param port [Integer, nil] bind port override
+    # @param sqlite_store [Mammoth::SQLiteStore, nil] optional operational store
+    # @param logger [WEBrick::Log, nil] optional WEBrick logger
+    def initialize(config, host: nil, port: nil, sqlite_store: nil, logger: nil)
+      @config = config
+      @host = host || config.dig("observability", "host") || DEFAULT_HOST
+      @port = port || config.dig("observability", "port") || DEFAULT_PORT
+      @sqlite_store = sqlite_store
+      @logger = logger || WEBrick::Log.new($stderr, WEBrick::Log::WARN)
+      @server = build_server
+      mount_endpoints
+    end
+
+    # Start the blocking HTTP server.
+    #
+    # @return [void]
+    def start
+      server.start
+    end
+
+    # Stop the HTTP server.
+    #
+    # @return [void]
+    def shutdown
+      server.shutdown
+    end
+
+    private
+
+    def build_server
+      WEBrick::HTTPServer.new(
+        BindAddress: host,
+        Port: port,
+        Logger: logger,
+        AccessLog: []
+      )
+    end
+
+    def mount_endpoints
+      server.mount_proc("/healthz") { |_request, response| write_json(response, snapshot.health, status: 200) }
+      server.mount_proc("/readyz") { |_request, response| write_readiness(response) }
+      server.mount_proc("/metrics") { |_request, response| write_metrics(response) }
+    end
+
+    def write_readiness(response)
+      payload = snapshot.readiness
+      write_json(response, payload, status: payload.fetch(:status) == "ready" ? 200 : 503)
+    end
+
+    def write_metrics(response)
+      response.status = 200
+      response["Content-Type"] = "text/plain; version=0.0.4; charset=utf-8"
+      response.body = snapshot.prometheus
+    end
+
+    def write_json(response, payload, status:)
+      response.status = status
+      response["Content-Type"] = "application/json"
+      response.body = JSON.generate(payload)
+    end
+
+    def snapshot
+      ObservabilitySnapshot.new(config, sqlite_store: sqlite_store)
+    end
+  end
+end

data/lib/mammoth/observability_snapshot.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Mammoth
+  # Builds health, readiness, and metrics snapshots from Mammoth's local
+  # operational state.
+  #
+  # ObservabilitySnapshot is intentionally read-only. It does not start the
+  # relay, mutate checkpoints, replay dead letters, or inspect PostgreSQL. The
+  # health and metrics endpoints use this object to expose Mammoth process and
+  # SQLite operational-state status in a predictable format.
+  class ObservabilitySnapshot
+    attr_reader :config, :sqlite_store, :clock
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param sqlite_store [Mammoth::SQLiteStore, nil] optional operational store
+    # @param clock [#call] time source returning a Time-like object
+    def initialize(config, sqlite_store: nil, clock: -> { Time.now.utc })
+      @config = config
+      @sqlite_store = sqlite_store
+      @clock = clock
+    end
+
+    # Build a liveness response.
+    #
+    # @return [Hash] health payload
+    def health
+      {
+        status: "ok",
+        service: "mammoth",
+        name: mammoth_name,
+        version: Mammoth::VERSION,
+        checked_at: checked_at
+      }
+    end
+
+    # Build a readiness response.
+    #
+    # @return [Hash] readiness payload
+    def readiness
+      store = operational_store
+      store.bootstrap!
+
+      {
+        status: "ready",
+        service: "mammoth",
+        name: mammoth_name,
+        sqlite: "ok",
+        tables: store.tables,
+        checked_at: checked_at
+      }
+    rescue Mammoth::Error, SQLite3::Exception => e
+      {
+        status: "unready",
+        service: "mammoth",
+        name: mammoth_name,
+        sqlite: "error",
+        error_class: e.class.name,
+        error_message: e.message,
+        checked_at: checked_at
+      }
+    end
+
+    # Build a Prometheus text exposition document.
+    #
+    # @return [String] Prometheus metrics text
+    def prometheus
+      store = operational_store.bootstrap!
+      checkpoint_store = CheckpointStore.new(store)
+      dead_letter_store = DeadLetterStore.new(store)
+      delivered_store = DeliveredEnvelopeStore.new(store)
+
+      lines = metric_headers + [
+        metric_line("mammoth_up", 1),
+        metric_line("mammoth_checkpoints_total", checkpoint_store.count),
+        metric_line("mammoth_dead_letters_total", dead_letter_store.count),
+        metric_line("mammoth_dead_letters_pending_total", dead_letter_store.count(status: "pending")),
+        metric_line("mammoth_dead_letters_resolved_total", dead_letter_store.count(status: "resolved")),
+        metric_line("mammoth_dead_letters_ignored_total", dead_letter_store.count(status: "ignored")),
+        metric_line("mammoth_delivered_envelopes_total", delivered_store.count)
+      ]
+      "#{lines.join("\n")}\n"
+    rescue Mammoth::Error, SQLite3::Exception
+      "#{(metric_headers + [metric_line("mammoth_up", 0)]).join("\n")}\n"
+    end
+
+    private
+
+    def operational_store
+      sqlite_store || SQLiteStore.connect(config.dig("sqlite", "path"))
+    end
+
+    def mammoth_name
+      config.dig("mammoth", "name")
+    end
+
+    def checked_at
+      clock.call.utc.iso8601
+    end
+
+    def metric_headers
+      [
+        "# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
+        "# TYPE mammoth_up gauge",
+        "# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
+        "# TYPE mammoth_checkpoints_total gauge",
+        "# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
+        "# TYPE mammoth_dead_letters_total gauge",
+        "# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
+        "# TYPE mammoth_dead_letters_pending_total gauge",
+        "# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
+        "# TYPE mammoth_dead_letters_resolved_total gauge",
+        "# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
+        "# TYPE mammoth_dead_letters_ignored_total gauge",
+        "# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
+        "# TYPE mammoth_delivered_envelopes_total gauge"
+      ]
+    end
+
+    def metric_line(name, value)
+      %(#{name}{mammoth_name="#{escape_label(mammoth_name)}"} #{Integer(value)})
+    end
+
+    def escape_label(value)
+      value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
+    end
+  end
+end

data/lib/mammoth/sources/postgres.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Mammoth
+  # Namespace for Mammoth source adapters.
   module Sources
     # Concrete PostgreSQL CDC source for Mammoth.
     #

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/mammoth/webhook_sink.rb

@@ -13,7 +13,9 @@ module Mammoth
     # HTTP status range treated as successful webhook delivery.
     SUCCESS_RANGE = 200..299
 
+    # Supported webhook signing algorithm.
     SIGNING_ALGORITHM = "hmac_sha256"
+    # Prefix added to generated webhook signatures.
     SIGNATURE_PREFIX = "sha256="
 
     attr_reader :name, :url, :timeout_seconds, :headers, :signing

data/lib/mammoth.rb

@@ -4,6 +4,8 @@ require_relative "mammoth/version"
 require_relative "mammoth/errors"
 require_relative "mammoth/configuration"
 require_relative "mammoth/status"
+require_relative "mammoth/observability_snapshot"
+require_relative "mammoth/observability_server"
 require_relative "mammoth/sqlite_store"
 require_relative "mammoth/checkpoint_store"
 require_relative "mammoth/dead_letter_store"
@@ -18,6 +20,7 @@ require_relative "mammoth/sources/postgres"
 require_relative "mammoth/cdc_source"
 require_relative "mammoth/replication_consumer"
 require_relative "mammoth/application"
+require_relative "mammoth/dead_letter_commands"
 require_relative "mammoth/cli"
 
 # Mammoth is a self-hosted PostgreSQL event relay.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mammoth
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -150,12 +150,15 @@ files:
 - lib/mammoth/cli.rb
 - lib/mammoth/concurrent_delivery_runtime.rb
 - lib/mammoth/configuration.rb
+- lib/mammoth/dead_letter_commands.rb
 - lib/mammoth/dead_letter_store.rb
 - lib/mammoth/delivered_envelope_store.rb
 - lib/mammoth/delivery_processor.rb
 - lib/mammoth/delivery_worker.rb
 - lib/mammoth/errors.rb
 - lib/mammoth/event_serializer.rb
+- lib/mammoth/observability_server.rb
+- lib/mammoth/observability_snapshot.rb
 - lib/mammoth/replication_consumer.rb
 - lib/mammoth/sources/postgres.rb
 - lib/mammoth/sql/__bootstrap__.sql