mammoth 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d7d570675d2b6731d783c6eba0f347ab00c1d7f9e524c20eef4908af129c41a
-  data.tar.gz: f14c3223b20cd1ea8e7d795e1cd2fe4e512766a0a4ed275515e4d1174bb3d6f5
+  metadata.gz: 8f299dc783af6453331a635200b8fe3ddf03159b156a3a0cec6cc6377ede1c34
+  data.tar.gz: f56eabc963912066610c22a98af55af9a6db192f40869dfb1d5743c8f1eff98e
 SHA512:
-  metadata.gz: cf649aba9068bdc1c3b0e01c1254f8f16804e80ec88de725d2743e99be9b9010bd880590fde044c9d7dff3b0b99fac54c6c9776af4504fb566eeceb1e5b00fd9
-  data.tar.gz: a7396f91f4fe84b3780f9b176931d728707b6e55ecab9b168b5eb3e87cb384c2066d9ed0b944dd14405d2e06a02e0e363a28348325578bb652aa4ba1df56caee
+  metadata.gz: c3167effc0333430719118b2ad3f8d8b53841c8c584b1e45949e256411e1a275eb56b71bd023d4d3778e31ec92f179c338d5814220f11c251f447aad7eee0233
+  data.tar.gz: d6db35472dfb2847e00f2158fd8677f26da5cb3c19d91e002566d598d9ecd8d67681e35dc2310ffd7fd41dcc0aec5535e822fd52f55f3865f9f65cc267633383

data/CHANGELOG.md

@@ -2,7 +2,21 @@
 
 ## Unreleased
 
-## 0.3.0
+## 0.5.0
+
+### Added
+
+- Added multi-destination webhook fanout through `destinations`, with independent per-destination retry, delivered-ledger, and dead-letter state.
+- Added targeted fanout dead-letter replay so replay sends failed work back only to the original destination.
+- Added Helm rendering for fanout destinations, including secret-backed environment variables for Authorization headers and HMAC signing secrets.
+
+## 0.4.0
+
+### Added
+
+- Added optional observability support with `/healthz`, `/readyz`, and `/metrics` endpoints, plus CLI startup, configuration, and documentation coverage.
+
+## 0.4.0
 
 ### Added
 

data/README.md

@@ -17,7 +17,7 @@ CDC::Core::TransactionEnvelope
       ↓
 Mammoth
       ↓
-Webhook
+Webhook fanout
 ```
 
 🦣 Mammoth is intentionally boring infrastructure. It uses YAML configuration,
@@ -46,6 +46,7 @@ https://kanutocd.github.io/mammoth/Mammoth.html
 - checkpoint persistence
 - dead letter persistence
 - webhook delivery sink
+- webhook fanout to multiple destinations
 - delivery worker with retry, checkpoint, and DLQ handling
 - dead-letter inspection and replay commands
 - CDC-core event serialization boundary
@@ -53,10 +54,11 @@ https://kanutocd.github.io/mammoth/Mammoth.html
 - Docker image support
 - public Helm chart support
 - unit and e2e test tasks
+- health and metrics endpoints
 
 ## Boundary
 
-Mammoth begins at CDC-core work items and ends at webhook delivery.
+Mammoth begins at CDC-core work items and ends at webhook fanout delivery.
 
 Mammoth does not own pgoutput protocol parsing, value decoding, source
 normalization, ordering policy, or runtime execution. Those belong to the
@@ -84,6 +86,7 @@ bundle exec ./exe/mammoth validate config/mammoth.example.yml
 bundle exec ./exe/mammoth bootstrap config/mammoth.example.yml
 bundle exec ./exe/mammoth status config/mammoth.example.yml
 bundle exec ./exe/mammoth start config/mammoth.example.yml
+bundle exec ./exe/mammoth observability config/mammoth.example.yml
 ```
 
 Deliver a single normalized event JSON file through Mammoth's delivery path:
@@ -101,6 +104,7 @@ Mammoth stores operational memory in SQLite:
 - `schema_migrations`
 - `checkpoints`
 - `dead_letters`
+- `delivered_envelopes`
 
 ## Performance
 

data/config/mammoth.example.yml

@@ -2,9 +2,10 @@
 
 # Mammoth example configuration.
 #
-# This file is intentionally verbose. It is both a runnable starting point and
-# the primary operator-facing documentation for local development, Kubernetes
-# values, and production deployments.
+# This file is intentionally verbose. It is the primary operator-facing
+# reference for local development, Kubernetes values, and production
+# deployments. Update URLs, environment variables, and PostgreSQL settings
+# before using it as a runtime config.
 #
 # Architecture reminder:
 #
@@ -24,7 +25,7 @@
 mammoth:
   # Human-readable Mammoth instance name.
   #
-  # Used in logs, status output, metrics, and future operational dashboards.
+  # Used in logs, status output, metrics, and operational dashboards.
   # Keep this stable per environment.
   #
   # Examples:
@@ -177,8 +178,7 @@ runtime:
   #
   # concurrent:
   #   Use CDC::Concurrent::ProcessorPool for I/O-heavy delivery workloads such as
-  #   webhook HTTP calls, retry sleeps/backoff, and future multi-destination
-  #   fanout.
+  #   webhook HTTP calls, retry sleeps/backoff, and multi-destination fanout.
   adapter: concurrent
 
   # Number of downstream delivery workers.
@@ -193,6 +193,12 @@ runtime:
   #   webhook-heavy production: 10-50, after soak testing
   concurrency: 1
 
+  # Number of work units submitted to the runtime together.
+  #
+  # Keep this at 1 unless you have validated downstream ordering, retry, and
+  # receiver idempotency behavior under load.
+  batch_size: 1
+
   # Preserve configured ordering while using the runtime.
   #
   # true:
@@ -206,11 +212,17 @@ runtime:
   # Optional runtime timeout in seconds.
   #
   # Leave blank to let destination-specific timeouts control delivery behavior.
-  # Useful later for bounding end-to-end processor execution.
+  # Use this to bound end-to-end processor execution.
   timeout_seconds:
 
+# Single webhook destination shorthand.
+#
+# Use this for one destination. Use destinations below instead when Mammoth
+# should fan out to multiple webhook receivers. Do not configure both unless you
+# intentionally want destinations to replace this shorthand.
 webhook:
-  # Destination name used in logs, dead-letter records, and future metrics.
+  # Destination name used in logs, dead-letter records, delivered ledgers, and
+  # metrics.
   name: primary_webhook
 
   # Destination URL for delivered CDC payloads.
@@ -230,6 +242,7 @@ webhook:
     X-Mammoth-Source: local_mammoth
 
   # Optional HTTP headers whose values are read from environment variables.
+  # In Kubernetes, set these variables from Secrets.
   #
   # Example:
   #   header_env:
@@ -244,6 +257,7 @@ webhook:
   #
   # using the secret from secret_env, then sends the timestamp and signature in
   # configurable headers. The signature value is prefixed with "sha256=".
+  # In Kubernetes, set the secret_env variable from a Secret.
   #
   # signing:
   #   algorithm: hmac_sha256
@@ -251,6 +265,28 @@ webhook:
   #   signature_header: X-Mammoth-Signature
   #   timestamp_header: X-Mammoth-Timestamp
 
+# Optional fanout configuration.
+#
+# Use destinations instead of webhook when one PostgreSQL replication stream
+# should deliver the same CDC work item to multiple webhook receivers. Each
+# destination gets independent retry, delivered-ledger, and dead-letter state.
+# The destinations list replaces the webhook shorthand above.
+#
+# destinations:
+#   - name: primary_webhook
+#     type: webhook
+#     url: https://example.com/webhooks/postgres
+#     timeout_seconds: 5
+#     header_env:
+#       Authorization: MAMMOTH_PRIMARY_WEBHOOK_AUTHORIZATION
+#     signing:
+#       algorithm: hmac_sha256
+#       secret_env: MAMMOTH_PRIMARY_WEBHOOK_SIGNING_SECRET
+#   - name: audit_webhook
+#     type: webhook
+#     url: https://audit.example.com/cdc
+#     timeout_seconds: 5
+
 retry:
   # Maximum delivery attempts before the event/envelope is moved to the dead
   # letter store.
@@ -270,10 +306,17 @@ retry:
 sqlite:
   # Local operational database path.
   #
-  # Mammoth stores checkpoints, delivery attempts, and dead-letter records here.
-  # In Kubernetes, back this path with a PersistentVolumeClaim.
+  # Mammoth stores checkpoints, delivered-envelope ledgers, and dead-letter
+  # records here. In Kubernetes, back this path with a PersistentVolumeClaim.
   path: data/mammoth.db
 
+observability:
+  # Bind host for the optional health, readiness, and metrics server.
+  host: 0.0.0.0
+
+  # HTTP port for /healthz, /readyz, and /metrics.
+  port: 9393
+
 logging:
   # Log verbosity.
   #

data/config/mammoth.schema.json

@@ -8,11 +8,22 @@
     "mammoth",
     "postgres",
     "replication",
-    "webhook",
     "retry",
     "sqlite",
     "logging"
   ],
+  "anyOf": [
+    {
+      "required": [
+        "webhook"
+      ]
+    },
+    {
+      "required": [
+        "destinations"
+      ]
+    }
+  ],
   "properties": {
     "mammoth": {
       "type": "object",
@@ -187,7 +198,93 @@
           "description": "Optional HMAC-SHA256 request signing for webhook receivers."
         }
       },
-      "description": "HTTP webhook destination settings."
+      "description": "HTTP webhook destination settings. This is shorthand for a single webhook destination; use destinations for fanout."
+    },
+    "destinations": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "name",
+          "type",
+          "url",
+          "timeout_seconds"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Destination name used in delivered ledgers, dead-letter records, logs, and metrics."
+          },
+          "type": {
+            "type": "string",
+            "enum": [
+              "webhook"
+            ],
+            "description": "Destination adapter type. Mammoth OSS 0.5.0 supports webhook destinations."
+          },
+          "url": {
+            "type": "string",
+            "format": "uri",
+            "description": "Webhook URL for delivered CDC payloads. Transaction delivery sends a TransactionEnvelope payload; event delivery sends a single event payload."
+          },
+          "timeout_seconds": {
+            "type": "integer",
+            "minimum": 1,
+            "description": "HTTP timeout in seconds for this destination's delivery attempts."
+          },
+          "headers": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "string"
+            },
+            "description": "Static HTTP headers added to every request for this destination. Do not put secrets here; use header_env for secret-backed values."
+          },
+          "header_env": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "string",
+              "minLength": 1
+            },
+            "description": "HTTP headers whose values are read from environment variables. Keys are header names; values are environment variable names."
+          },
+          "signing": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "secret_env"
+            ],
+            "properties": {
+              "algorithm": {
+                "type": "string",
+                "enum": [
+                  "hmac_sha256"
+                ],
+                "description": "Webhook signing algorithm. Mammoth signs '<timestamp>.<body>' using HMAC-SHA256."
+              },
+              "secret_env": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Environment variable containing the webhook signing secret."
+              },
+              "signature_header": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Header name used for the generated signature. Defaults to X-Mammoth-Signature."
+              },
+              "timestamp_header": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Header name used for the signing timestamp. Defaults to X-Mammoth-Timestamp."
+              }
+            },
+            "description": "Optional HMAC-SHA256 request signing for this webhook destination."
+          }
+        }
+      },
+      "description": "Fanout destination list. Each CDC work item is delivered independently to each configured destination."
     },
     "retry": {
       "type": "object",
@@ -319,6 +416,24 @@
         }
       },
       "description": "Downstream delivery runtime settings. Runtime concurrency affects webhook/destination delivery only, not PostgreSQL replication stream count."
+    },
+    "observability": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "host": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Bind host for the optional Mammoth observability HTTP server exposing /healthz, /readyz, and /metrics."
+        },
+        "port": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 65535,
+          "description": "TCP port for the optional Mammoth observability HTTP server."
+        }
+      },
+      "description": "Optional health, readiness, and metrics endpoint settings."
     }
   },
   "description": "Mammoth relay configuration. Mammoth consumes PostgreSQL logical replication changes and delivers them to downstream destinations while preserving checkpoint, retry, and dead-letter semantics."

data/lib/mammoth/application.rb

@@ -21,7 +21,7 @@ module Mammoth
       @sqlite_store = SQLiteStore.connect(config.dig("sqlite", "path")).bootstrap!
       @checkpoint_store = CheckpointStore.new(sqlite_store)
       @consumer = ReplicationConsumer.new(source: source || build_source, delivery_unit: delivery_unit)
-      @delivery_worker = build_delivery_worker(sink: sink || WebhookSink.from_config(config), sleeper: sleeper)
+      @delivery_worker = sink ? build_delivery_worker(sink: sink, sleeper: sleeper) : build_configured_delivery_worker(sleeper:)
     end
 
     # Start the application runtime and deliver consumed CDC work.
@@ -97,6 +97,22 @@ module Mammoth
       )
     end
 
+    def build_configured_delivery_worker(sleeper:)
+      workers = destination_sinks.map { |sink| build_delivery_worker(sink:, sleeper:) }
+      return workers.fetch(0) if workers.one?
+
+      FanoutDeliveryWorker.new(workers)
+    end
+
+    def destination_sinks
+      destinations = config.data["destinations"]
+      return [WebhookSink.from_config(config)] unless destinations
+
+      destinations.map.with_index(1) do |destination, index|
+        WebhookSink.from_destination_config(destination, label: "destinations[#{index - 1}]")
+      end
+    end
+
     def transaction_delivery?
       delivery_unit == :transaction
     end

data/lib/mammoth/cli.rb

@@ -26,7 +26,8 @@ module Mammoth
       "  mammoth deliver-sample CONFIG EVENT_JSON",
       "  mammoth dead-letters list CONFIG [--status STATUS] [--limit N]",
       "  mammoth dead-letters show CONFIG ID",
-      "  mammoth dead-letters replay CONFIG [ID ...]"
+      "  mammoth dead-letters replay CONFIG [ID ...]",
+      "  mammoth observability CONFIG"
     ].join("\n")
 
     # Run the CLI.
@@ -56,6 +57,7 @@ module Mammoth
       when "start" then start
       when "deliver-sample" then deliver_sample
       when "dead-letters" then dead_letters
+      when "observability" then observability
       else
         warn USAGE
         1
@@ -130,6 +132,14 @@ module Mammoth
       DeadLetterCommands.call(argv)
     end
 
+    def observability
+      config = load_config
+      server = ObservabilityServer.new(config)
+      puts "Mammoth observability listening on #{server.host}:#{server.port}"
+      server.start
+      0
+    end
+
     def load_config
       raise ConfigurationError, "configuration path required\n#{USAGE}" unless config_path
 

data/lib/mammoth/configuration.rb

@@ -44,6 +44,7 @@ module Mammoth
       raise ConfigurationError, "configuration must be a YAML mapping" unless data.is_a?(Hash)
 
       validate_schema!
+      validate_destination_names!
       self
     rescue Psych::SyntaxError => e
       raise ConfigurationError, "invalid YAML in #{path}: #{e.message}"
@@ -73,5 +74,16 @@ module Mammoth
     def schema_error_message(errors)
       (["configuration failed schema validation:"] + errors.map { |error| "- #{error}" }).join("\n")
     end
+
+    def validate_destination_names!
+      destinations = data["destinations"]
+      return unless destinations
+
+      names = destinations.map { |destination| destination["name"] }
+      duplicates = names.tally.select { |_name, count| count > 1 }.keys
+      return if duplicates.empty?
+
+      raise ConfigurationError, "destination names must be unique: #{duplicates.join(", ")}"
+    end
   end
 end

data/lib/mammoth/dead_letter_commands.rb

@@ -163,7 +163,23 @@ module Mammoth
 
     def replay_row(row)
       payload = JSON.parse(row.fetch("payload_json"))
-      transaction_payload?(payload) ? worker.deliver_transaction(transaction_envelope(payload)) : worker.deliver(payload)
+      if transaction_payload?(payload)
+        replay_transaction(row.fetch("destination_name"), transaction_envelope(payload))
+      else
+        replay_event(row.fetch("destination_name"), payload)
+      end
+    end
+
+    def replay_event(destination_name, payload)
+      return worker.deliver_to(destination_name, payload) if worker.respond_to?(:deliver_to)
+
+      worker.deliver(payload)
+    end
+
+    def replay_transaction(destination_name, envelope)
+      return worker.deliver_transaction_to(destination_name, envelope) if worker.respond_to?(:deliver_transaction_to)
+
+      worker.deliver_transaction(envelope)
     end
 
     def transaction_payload?(payload)

data/lib/mammoth/fanout_delivery_worker.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Delivers one CDC work item to multiple destination-specific workers.
+  class FanoutDeliveryWorker
+    attr_reader :delivery_workers
+
+    # @param delivery_workers [Array<Mammoth::DeliveryWorker>] destination workers
+    def initialize(delivery_workers)
+      raise ConfigurationError, "at least one destination is required" if delivery_workers.empty?
+
+      @delivery_workers = delivery_workers
+    end
+
+    # Deliver an event to every configured destination.
+    #
+    # @param event [Hash, #to_h] normalized event
+    # @return [Hash] aggregate fanout summary
+    def deliver(event)
+      fanout(:deliver, event)
+    end
+
+    # Deliver a transaction envelope to every configured destination.
+    #
+    # @param envelope [#events, #transaction_id] CDC transaction envelope
+    # @return [Hash] aggregate fanout summary
+    def deliver_transaction(envelope)
+      fanout(:deliver_transaction, envelope)
+    end
+
+    # Deliver an event to one configured destination.
+    #
+    # @param destination_name [String] destination name
+    # @param event [Hash, #to_h] normalized event
+    # @return [Hash] destination delivery summary
+    def deliver_to(destination_name, event)
+      worker_for(destination_name).deliver(event)
+    end
+
+    # Deliver a transaction envelope to one configured destination.
+    #
+    # @param destination_name [String] destination name
+    # @param envelope [#events, #transaction_id] CDC transaction envelope
+    # @return [Hash] destination delivery summary
+    def deliver_transaction_to(destination_name, envelope)
+      worker_for(destination_name).deliver_transaction(envelope)
+    end
+
+    private
+
+    def fanout(delivery_method, work)
+      results = delivery_workers.map { |worker| worker.public_send(delivery_method, work) }
+      {
+        status: aggregate_status(results),
+        destinations: results,
+        delivered: results.count { |result| result.fetch(:status) == "delivered" },
+        skipped: results.count { |result| result.fetch(:status) == "skipped" },
+        dead_lettered: results.count { |result| result.fetch(:status) == "dead_lettered" }
+      }
+    end
+
+    def aggregate_status(results)
+      return "fanout_delivered" if results.all? { |result| %w[delivered skipped].include?(result.fetch(:status)) }
+
+      "fanout_partial"
+    end
+
+    def worker_for(destination_name)
+      delivery_workers.find { |worker| worker.send(:destination_name) == destination_name } ||
+        raise(ConfigurationError, "destination not configured: #{destination_name}")
+    end
+  end
+end

data/lib/mammoth/observability_server.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "json"
+require "webrick"
+
+module Mammoth
+  # Small HTTP server exposing Mammoth health, readiness, and metrics endpoints.
+  #
+  # The server is intentionally independent from the replication loop. Operators
+  # may run it as a sidecar-like process or in a separate process that points at
+  # the same SQLite operational database.
+  class ObservabilityServer
+    DEFAULT_HOST = "0.0.0.0"
+    DEFAULT_PORT = 9393
+
+    attr_reader :config, :host, :port, :sqlite_store, :logger, :server
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param host [String, nil] bind host override
+    # @param port [Integer, nil] bind port override
+    # @param sqlite_store [Mammoth::SQLiteStore, nil] optional operational store
+    # @param logger [WEBrick::Log, nil] optional WEBrick logger
+    def initialize(config, host: nil, port: nil, sqlite_store: nil, logger: nil)
+      @config = config
+      @host = host || config.dig("observability", "host") || DEFAULT_HOST
+      @port = port || config.dig("observability", "port") || DEFAULT_PORT
+      @sqlite_store = sqlite_store
+      @logger = logger || WEBrick::Log.new($stderr, WEBrick::Log::WARN)
+      @server = build_server
+      mount_endpoints
+    end
+
+    # Start the blocking HTTP server.
+    #
+    # @return [void]
+    def start
+      server.start
+    end
+
+    # Stop the HTTP server.
+    #
+    # @return [void]
+    def shutdown
+      server.shutdown
+    end
+
+    private
+
+    def build_server
+      WEBrick::HTTPServer.new(
+        BindAddress: host,
+        Port: port,
+        Logger: logger,
+        AccessLog: []
+      )
+    end
+
+    def mount_endpoints
+      server.mount_proc("/healthz") { |_request, response| write_json(response, snapshot.health, status: 200) }
+      server.mount_proc("/readyz") { |_request, response| write_readiness(response) }
+      server.mount_proc("/metrics") { |_request, response| write_metrics(response) }
+    end
+
+    def write_readiness(response)
+      payload = snapshot.readiness
+      write_json(response, payload, status: payload.fetch(:status) == "ready" ? 200 : 503)
+    end
+
+    def write_metrics(response)
+      response.status = 200
+      response["Content-Type"] = "text/plain; version=0.0.4; charset=utf-8"
+      response.body = snapshot.prometheus
+    end
+
+    def write_json(response, payload, status:)
+      response.status = status
+      response["Content-Type"] = "application/json"
+      response.body = JSON.generate(payload)
+    end
+
+    def snapshot
+      ObservabilitySnapshot.new(config, sqlite_store: sqlite_store)
+    end
+  end
+end

data/lib/mammoth/observability_snapshot.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Mammoth
+  # Builds health, readiness, and metrics snapshots from Mammoth's local
+  # operational state.
+  #
+  # ObservabilitySnapshot is intentionally read-only. It does not start the
+  # relay, mutate checkpoints, replay dead letters, or inspect PostgreSQL. The
+  # health and metrics endpoints use this object to expose Mammoth process and
+  # SQLite operational-state status in a predictable format.
+  class ObservabilitySnapshot
+    attr_reader :config, :sqlite_store, :clock
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @param sqlite_store [Mammoth::SQLiteStore, nil] optional operational store
+    # @param clock [#call] time source returning a Time-like object
+    def initialize(config, sqlite_store: nil, clock: -> { Time.now.utc })
+      @config = config
+      @sqlite_store = sqlite_store
+      @clock = clock
+    end
+
+    # Build a liveness response.
+    #
+    # @return [Hash] health payload
+    def health
+      {
+        status: "ok",
+        service: "mammoth",
+        name: mammoth_name,
+        version: Mammoth::VERSION,
+        checked_at: checked_at
+      }
+    end
+
+    # Build a readiness response.
+    #
+    # @return [Hash] readiness payload
+    def readiness
+      store = operational_store
+      store.bootstrap!
+
+      {
+        status: "ready",
+        service: "mammoth",
+        name: mammoth_name,
+        sqlite: "ok",
+        tables: store.tables,
+        checked_at: checked_at
+      }
+    rescue Mammoth::Error, SQLite3::Exception => e
+      {
+        status: "unready",
+        service: "mammoth",
+        name: mammoth_name,
+        sqlite: "error",
+        error_class: e.class.name,
+        error_message: e.message,
+        checked_at: checked_at
+      }
+    end
+
+    # Build a Prometheus text exposition document.
+    #
+    # @return [String] Prometheus metrics text
+    def prometheus
+      store = operational_store.bootstrap!
+      checkpoint_store = CheckpointStore.new(store)
+      dead_letter_store = DeadLetterStore.new(store)
+      delivered_store = DeliveredEnvelopeStore.new(store)
+
+      lines = metric_headers + [
+        metric_line("mammoth_up", 1),
+        metric_line("mammoth_checkpoints_total", checkpoint_store.count),
+        metric_line("mammoth_dead_letters_total", dead_letter_store.count),
+        metric_line("mammoth_dead_letters_pending_total", dead_letter_store.count(status: "pending")),
+        metric_line("mammoth_dead_letters_resolved_total", dead_letter_store.count(status: "resolved")),
+        metric_line("mammoth_dead_letters_ignored_total", dead_letter_store.count(status: "ignored")),
+        metric_line("mammoth_delivered_envelopes_total", delivered_store.count)
+      ]
+      "#{lines.join("\n")}\n"
+    rescue Mammoth::Error, SQLite3::Exception
+      "#{(metric_headers + [metric_line("mammoth_up", 0)]).join("\n")}\n"
+    end
+
+    private
+
+    def operational_store
+      sqlite_store || SQLiteStore.connect(config.dig("sqlite", "path"))
+    end
+
+    def mammoth_name
+      config.dig("mammoth", "name")
+    end
+
+    def checked_at
+      clock.call.utc.iso8601
+    end
+
+    def metric_headers
+      [
+        "# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
+        "# TYPE mammoth_up gauge",
+        "# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
+        "# TYPE mammoth_checkpoints_total gauge",
+        "# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
+        "# TYPE mammoth_dead_letters_total gauge",
+        "# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
+        "# TYPE mammoth_dead_letters_pending_total gauge",
+        "# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
+        "# TYPE mammoth_dead_letters_resolved_total gauge",
+        "# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
+        "# TYPE mammoth_dead_letters_ignored_total gauge",
+        "# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
+        "# TYPE mammoth_delivered_envelopes_total gauge"
+      ]
+    end
+
+    def metric_line(name, value)
+      %(#{name}{mammoth_name="#{escape_label(mammoth_name)}"} #{Integer(value)})
+    end
+
+    def escape_label(value)
+      value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
+    end
+  end
+end

data/lib/mammoth/status.rb

@@ -30,7 +30,7 @@ module Mammoth
       puts "Replication publications: #{Array(config.dig("replication", "publications")).join(", ")}"
       puts "Runtime: not started"
       puts "SQLite: #{sqlite_path}"
-      puts "Webhook: #{config.dig("webhook", "name")}"
+      puts "Destinations: #{destination_names.join(", ")}"
       print_store_state if sqlite_store
     end
 
@@ -40,6 +40,13 @@ module Mammoth
       config.dig("sqlite", "path")
     end
 
+    def destination_names
+      destinations = config.data["destinations"]
+      return destinations.map { |destination| destination.fetch("name") } if destinations
+
+      [config.dig("webhook", "name")]
+    end
+
     def print_store_state
       store = sqlite_store.bootstrap!
       puts "Tables: #{store.tables.join(", ")}"

data/lib/mammoth/version.rb

@@ -2,5 +2,5 @@
 
 module Mammoth
   # Current Mammoth gem version.
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/mammoth/webhook_sink.rb

@@ -39,43 +39,52 @@ module Mammoth
       # @param config [Mammoth::Configuration] loaded configuration
       # @return [Mammoth::WebhookSink]
       def from_config(config)
+        from_destination_config(config.data["webhook"], label: "webhook")
+      end
+
+      # Build a sink from one destination configuration entry.
+      #
+      # @param destination [Hash] destination configuration
+      # @param label [String] configuration path used in error messages
+      # @return [Mammoth::WebhookSink]
+      def from_destination_config(destination, label: "destination")
         new(
-          name: config.dig("webhook", "name"),
-          url: config.dig("webhook", "url"),
-          timeout_seconds: config.dig("webhook", "timeout_seconds"),
-          headers: configured_headers(config),
-          signing: configured_signing(config)
+          name: destination.fetch("name"),
+          url: destination.fetch("url"),
+          timeout_seconds: destination.fetch("timeout_seconds"),
+          headers: configured_headers(destination, label: label),
+          signing: configured_signing(destination, label: label)
         )
       end
 
       private
 
-      def configured_headers(config)
-        static_headers = config.dig("webhook", "headers") || {}
-        env_headers = config.dig("webhook", "header_env") || {}
+      def configured_headers(destination, label:)
+        static_headers = destination["headers"] || {}
+        env_headers = destination["header_env"] || {}
 
-        static_headers.merge(resolve_env_headers(env_headers))
+        static_headers.merge(resolve_env_headers(env_headers, label: label))
       end
 
-      def resolve_env_headers(env_headers)
+      def resolve_env_headers(env_headers, label:)
         env_headers.each_with_object(Hash.new) do |(header, env_name), resolved| # rubocop:disable Style/EmptyLiteral
           resolved[header] = ENV.fetch(env_name) do
-            raise ConfigurationError, "webhook.header_env.#{header} references missing environment variable #{env_name}"
+            raise ConfigurationError, "#{label}.header_env.#{header} references missing environment variable #{env_name}"
           end
         end
       end
 
-      def configured_signing(config)
-        signing = config.dig("webhook", "signing")
+      def configured_signing(destination, label:)
+        signing = destination["signing"]
         return unless signing
 
         algorithm = signing.fetch("algorithm", SIGNING_ALGORITHM)
-        raise ConfigurationError, "webhook.signing.algorithm must be #{SIGNING_ALGORITHM}" unless algorithm == SIGNING_ALGORITHM
+        raise ConfigurationError, "#{label}.signing.algorithm must be #{SIGNING_ALGORITHM}" unless algorithm == SIGNING_ALGORITHM
 
         secret_env = signing.fetch("secret_env")
         {
           secret: ENV.fetch(secret_env) do
-            raise ConfigurationError, "webhook.signing.secret_env references missing environment variable #{secret_env}"
+            raise ConfigurationError, "#{label}.signing.secret_env references missing environment variable #{secret_env}"
           end,
           signature_header: signing.fetch("signature_header", "X-Mammoth-Signature"),
           timestamp_header: signing.fetch("timestamp_header", "X-Mammoth-Timestamp")

data/lib/mammoth.rb

@@ -4,6 +4,8 @@ require_relative "mammoth/version"
 require_relative "mammoth/errors"
 require_relative "mammoth/configuration"
 require_relative "mammoth/status"
+require_relative "mammoth/observability_snapshot"
+require_relative "mammoth/observability_server"
 require_relative "mammoth/sqlite_store"
 require_relative "mammoth/checkpoint_store"
 require_relative "mammoth/dead_letter_store"
@@ -12,6 +14,7 @@ require_relative "mammoth/event_serializer"
 require_relative "mammoth/transaction_envelope_serializer"
 require_relative "mammoth/webhook_sink"
 require_relative "mammoth/delivery_worker"
+require_relative "mammoth/fanout_delivery_worker"
 require_relative "mammoth/delivery_processor"
 require_relative "mammoth/concurrent_delivery_runtime"
 require_relative "mammoth/sources/postgres"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mammoth
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Ken C. Demanawa
@@ -157,6 +157,9 @@ files:
 - lib/mammoth/delivery_worker.rb
 - lib/mammoth/errors.rb
 - lib/mammoth/event_serializer.rb
+- lib/mammoth/fanout_delivery_worker.rb
+- lib/mammoth/observability_server.rb
+- lib/mammoth/observability_snapshot.rb
 - lib/mammoth/replication_consumer.rb
 - lib/mammoth/sources/postgres.rb
 - lib/mammoth/sql/__bootstrap__.sql