mammoth 0.5.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c04c055c5cf5faedef6100513bc63a9ada3c69b99b904f701d0be93501aa4ebd
-  data.tar.gz: 15e75ed7ed39723f5f0864253196a830ae0413d6609ae504640be2eca7d6b2da
+  metadata.gz: 5fe6f64a39d214c62e11a29fccb091f58cd2a1b8038a0bb803c39c4e018b09b3
+  data.tar.gz: f557c7288df617029808c327b656a8b7cb1f7689eb7f4b6d99032b01bcc0535e
 SHA512:
-  metadata.gz: 1d0b5446e4b5edd10f5d82007dd8ae85e7e9e40f056725ef1faa5e090c2fa844a9007c94c7ac47b41cb7eeec6fe8d9be3eb0dec7685a757da749ff7dbf85ccbd
-  data.tar.gz: 3f3185bc9743f100bab26a62f2fa45f94f04aa3243e787ab05d0b13aafc697a22ab432a84da1e40dfd7214e880bafe727e17c8ca4e15236ca97547f55166687c
+  metadata.gz: fe49156b57ff2355fd0e78368cd613fa3fa5dcf5d0a5ebf423ec799558b61a70da7529a123f5b254b904a93e7c7355a049b115308eb051e211894b84443f3444
+  data.tar.gz: 7b24591aa6f3996e51b5762022f680f212cbb1e8412e0e90ff42e34c3485d9f18fb2f4f4c9b9ca4420a697767400a2d37b6ebc600d203e7c4f5f0fbe5e904da2

data/CHANGELOG.md

@@ -1,6 +1,34 @@
 # Changelog
 
-## Unreleased
+## 0.7.0 Unreleased
+
+### Added
+
+- Added explicit adapter registries for operational state, destinations, and delivery runtimes.
+- Added the built-in SQLite operational state adapter.
+- Added the built-in webhook destination adapter registration.
+- Added inline and concurrent runtime adapter registration.
+- Added optional node identity configuration for future control-plane integration.
+- Added local capability reporting for state, destination, runtime, and relay features.
+- Added a reusable status command object behind the CLI status command.
+
+### Changed
+
+- Bumped Mammoth version to `0.7.0`.
+
+## 0.6.0
+
+### Added
+
+- Added fanout route filters by schema, table, and operation for webhook destinations.
+- Added destination `enabled` controls for config-driven delivery cutovers.
+- Added per-destination retry policy overrides for `max_attempts` and `schedule_seconds`.
+- Added dead-letter replay filters by destination, status, and failed-at time window.
+- Added destination-labeled Prometheus metrics for dead-letter and delivered-envelope counts.
+
+### Changed
+
+- Bumped Mammoth version to `0.6.0`.
 
 ## 0.5.1
 

data/README.md

@@ -47,14 +47,18 @@ https://kanutocd.github.io/mammoth/Mammoth.html
 - dead letter persistence
 - webhook delivery sink
 - webhook fanout to multiple destinations
+- fanout route filters by schema, table, and operation
+- per-destination enable/disable and retry policy controls
 - delivery worker with retry, checkpoint, and DLQ handling
-- dead-letter inspection and replay commands
+- dead-letter inspection and filtered replay commands
 - CDC-core event serialization boundary
 - CDC Ecosystem source-adapter integration boundary
 - Docker image support
 - public Helm chart support
 - unit and e2e test tasks
 - health and metrics endpoints
+- explicit extension registries for state, destination, and runtime adapters
+- node identity and local capability reporting
 
 ## Boundary
 
@@ -64,6 +68,16 @@ Mammoth does not own pgoutput protocol parsing, value decoding, source
 normalization, ordering policy, or runtime execution. Those belong to the
 upstream CDC Ecosystem components.
 
+## Extensions
+
+Mammoth OSS exposes small adapter registries for future extensions:
+
+- operational state adapters
+- destination adapters
+- runtime adapters
+
+See [`docs/EXTENSIONS.md`](docs/EXTENSIONS.md).
+
 ## Configuration
 
 Mammoth configuration is YAML-backed and IDE-friendly.
@@ -78,6 +92,24 @@ Validate configuration:
 bundle exec ./exe/mammoth validate config/mammoth.example.yml
 ```
 
+Fanout destinations can be routed and tuned independently:
+
+```yaml
+destinations:
+  - name: audit_webhook
+    type: webhook
+    enabled: true
+    url: https://audit.example.com/cdc
+    timeout_seconds: 5
+    route:
+      schemas: [public]
+      tables: [orders]
+      operations: [insert, update]
+    retry:
+      max_attempts: 3
+      schedule_seconds: [1, 10]
+```
+
 ## CLI
 
 ```bash

data/config/mammoth.example.yml

@@ -34,6 +34,19 @@ mammoth:
   #   production_mammoth
   name: local_mammoth
 
+node:
+  # Optional local node identity for status output and future control-plane
+  # agents. Omit this section to default node_id to the host name and node_name
+  # to mammoth.name.
+  node_id: local-mammoth-1
+  node_name: local-mammoth-dev
+  fleet_id: local-dev
+  environment: development
+  labels:
+    region: local
+  metadata:
+    owner: platform
+
 postgres:
   # PostgreSQL host reachable from the Mammoth process or pod.
   #
@@ -275,6 +288,7 @@ webhook:
 # destinations:
 #   - name: primary_webhook
 #     type: webhook
+#     enabled: true
 #     url: https://example.com/webhooks/postgres
 #     timeout_seconds: 5
 #     header_env:
@@ -284,8 +298,28 @@ webhook:
 #       secret_env: MAMMOTH_PRIMARY_WEBHOOK_SIGNING_SECRET
 #   - name: audit_webhook
 #     type: webhook
+#     enabled: true
 #     url: https://audit.example.com/cdc
 #     timeout_seconds: 5
+#
+#     # Optional route filter. Omit route to receive every event/envelope.
+#     # For transaction delivery, a destination matches when any event in the
+#     # transaction matches; Mammoth still delivers the full transaction envelope.
+#     route:
+#       schemas:
+#         - public
+#       tables:
+#         - orders
+#       operations:
+#         - insert
+#         - update
+#
+#     # Optional per-destination retry override. Omit to use top-level retry.
+#     retry:
+#       max_attempts: 3
+#       schedule_seconds:
+#         - 1
+#         - 10
 
 retry:
   # Maximum delivery attempts before the event/envelope is moved to the dead
@@ -310,6 +344,10 @@ sqlite:
   # records here. In Kubernetes, back this path with a PersistentVolumeClaim.
   path: data/mammoth.db
 
+operational_state:
+  # Mammoth OSS 0.7.0 ships the sqlite operational state adapter.
+  adapter: sqlite
+
 observability:
   # Bind host for the optional health, readiness, and metrics server.
   host: 0.0.0.0

data/config/mammoth.schema.json

@@ -40,6 +40,47 @@
       },
       "description": "Mammoth instance identity and operator-facing metadata."
     },
+    "node": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "node_id": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Stable node identifier used by status output and future control-plane agents."
+        },
+        "node_name": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Human-readable node name."
+        },
+        "fleet_id": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Optional fleet identifier for grouping Mammoth nodes."
+        },
+        "environment": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Optional deployment environment such as development, staging, or production."
+        },
+        "labels": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          },
+          "description": "Operator-defined node labels."
+        },
+        "metadata": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          },
+          "description": "Operator-defined node metadata."
+        }
+      },
+      "description": "Optional local node identity. Mammoth OSS uses this for status and future extension surfaces only."
+    },
     "postgres": {
       "type": "object",
       "additionalProperties": false,
@@ -223,7 +264,11 @@
             "enum": [
               "webhook"
             ],
-            "description": "Destination adapter type. Mammoth OSS 0.5.1 supports webhook destinations."
+            "description": "Destination adapter type. Mammoth OSS 0.7.0 supports webhook destinations."
+          },
+          "enabled": {
+            "type": "boolean",
+            "description": "When false, Mammoth skips this destination, advances local progress for the work item, and does not attempt delivery."
           },
           "url": {
             "type": "string",
@@ -281,6 +326,61 @@
               }
             },
             "description": "Optional HMAC-SHA256 request signing for this webhook destination."
+          },
+          "route": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "schemas": {
+                "type": "array",
+                "minItems": 1,
+                "items": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "description": "PostgreSQL schemas this destination should receive. Omit to match every schema."
+              },
+              "tables": {
+                "type": "array",
+                "minItems": 1,
+                "items": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "description": "PostgreSQL tables this destination should receive. Omit to match every table."
+              },
+              "operations": {
+                "type": "array",
+                "minItems": 1,
+                "items": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "description": "CDC operations this destination should receive, such as insert, update, or delete. Omit to match every operation."
+              }
+            },
+            "description": "Optional destination route filter. Event delivery matches one event; transaction delivery matches when any event in the transaction matches and delivers the full envelope."
+          },
+          "retry": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "max_attempts": {
+                "type": "integer",
+                "minimum": 1,
+                "description": "Destination-specific maximum delivery attempts before dead-lettering. Defaults to the top-level retry.max_attempts."
+              },
+              "schedule_seconds": {
+                "type": "array",
+                "minItems": 1,
+                "items": {
+                  "type": "integer",
+                  "minimum": 1
+                },
+                "description": "Destination-specific retry backoff schedule in seconds. Defaults to the top-level retry.schedule_seconds."
+              }
+            },
+            "description": "Optional retry policy override for this destination."
           }
         }
       },
@@ -326,6 +426,20 @@
       },
       "description": "SQLite operational store settings for checkpoints, attempts, and dead-letter records."
     },
+    "operational_state": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "adapter": {
+          "type": "string",
+          "enum": [
+            "sqlite"
+          ],
+          "description": "Operational state adapter. Mammoth OSS 0.7.0 ships the sqlite adapter."
+        }
+      },
+      "description": "Operational state adapter selection. SQLite remains the default OSS adapter."
+    },
     "logging": {
       "type": "object",
       "additionalProperties": false,

data/lib/mammoth/application.rb

@@ -10,7 +10,7 @@ module Mammoth
   # injected CDC work source rather than owning upstream CDC source-adapter
   # lifecycle decisions.
   class Application
-    attr_reader :config, :sqlite_store, :consumer, :delivery_worker, :checkpoint_store
+    attr_reader :config, :state_adapter, :consumer, :delivery_worker, :checkpoint_store
 
     # @param config [Mammoth::Configuration] loaded configuration
     # @param source [#each, nil] injectable event source for tests and demos
@@ -18,12 +18,17 @@ module Mammoth
     # @param sleeper [#call] retry sleep strategy
     def initialize(config, source: nil, sink: nil, sleeper: Kernel.method(:sleep))
       @config = config
-      @sqlite_store = SQLiteStore.connect(config.dig("sqlite", "path")).bootstrap!
-      @checkpoint_store = CheckpointStore.new(sqlite_store)
+      @state_adapter = build_state_adapter
+      @checkpoint_store = state_adapter.checkpoint_store
       @consumer = ReplicationConsumer.new(source: source || build_source, delivery_unit: delivery_unit)
       @delivery_worker = sink ? build_delivery_worker(sink: sink, sleeper: sleeper) : build_configured_delivery_worker(sleeper:)
     end
 
+    # @return [Mammoth::SQLiteStore] underlying SQLite store for compatibility
+    def sqlite_store
+      state_adapter.respond_to?(:sqlite_store) ? state_adapter.sqlite_store : nil
+    end
+
     # Start the application runtime and deliver consumed CDC work.
     #
     # @return [Integer] number of processed work units
@@ -54,13 +59,7 @@ module Mammoth
     private
 
     def process_work(runtime, work)
-      if runtime
-        runtime.process_many([work])
-      elsif transaction_delivery?
-        delivery_worker.deliver_transaction(work)
-      else
-        delivery_worker.deliver(work)
-      end
+      runtime.process_many([work])
     end
 
     def process_batch(runtime, batch)
@@ -73,9 +72,8 @@ module Mammoth
     end
 
     def build_runtime
-      return unless runtime_adapter == "concurrent"
-
-      ConcurrentDeliveryRuntime.new(
+      Runtimes::Registry.build(
+        runtime_adapter,
         processor: DeliveryProcessor.new(delivery_worker:, delivery_unit: delivery_unit),
         concurrency: runtime_concurrency,
         timeout: runtime_timeout,
@@ -87,34 +85,57 @@ module Mammoth
       Sources::Postgres.new(config, checkpoint_store: checkpoint_store)
     end
 
-    def build_delivery_worker(sink:, sleeper:)
+    def build_delivery_worker(sink:, sleeper:, delivery_policy: {})
       DeliveryWorker.from_config(
         config,
         sink: sink,
         checkpoint_store: checkpoint_store,
-        dead_letter_store: DeadLetterStore.new(sqlite_store),
-        sleeper: sleeper
+        dead_letter_store: state_adapter.dead_letter_store,
+        sleeper: sleeper,
+        delivery_policy: delivery_policy
       )
     end
 
     def build_configured_delivery_worker(sleeper:)
-      workers = destination_sinks.map { |sink| build_delivery_worker(sink:, sleeper:) }
+      workers = destination_specs.map do |spec|
+        build_delivery_worker(sink: spec.fetch(:sink), sleeper:, delivery_policy: spec.fetch(:delivery_policy))
+      end
       return workers.fetch(0) if workers.one?
 
       FanoutDeliveryWorker.new(workers)
     end
 
-    def destination_sinks
+    def destination_specs
       destinations = config.data["destinations"]
-      return [WebhookSink.from_config(config)] unless destinations
+      unless destinations
+        delivery_policy = {} # : Hash[String, untyped]
+        return [{ sink: WebhookSink.from_config(config), delivery_policy: delivery_policy }]
+      end
 
       destinations.map.with_index(1) do |destination, index|
-        WebhookSink.from_destination_config(destination, label: "destinations[#{index - 1}]")
+        {
+          sink: Destinations::Registry.build(destination, label: "destinations[#{index - 1}]"),
+          delivery_policy: destination_delivery_policy(destination)
+        }
       end
     end
 
-    def transaction_delivery?
-      delivery_unit == :transaction
+    def destination_delivery_policy(destination)
+      retry_config = destination["retry"] || {}
+      {
+        "enabled" => destination.fetch("enabled", true),
+        "max_attempts" => retry_config.fetch("max_attempts", config.dig("retry", "max_attempts")),
+        "schedule_seconds" => retry_config.fetch("schedule_seconds", config.dig("retry", "schedule_seconds")),
+        "route_filter" => RouteFilter.new(destination["route"])
+      }
+    end
+
+    def build_state_adapter
+      OperationalState::Registry.build(operational_state_adapter, config)
+    end
+
+    def operational_state_adapter
+      config.dig("operational_state", "adapter") || "sqlite"
     end
 
     def delivery_unit

data/lib/mammoth/capabilities.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Reports local Mammoth capabilities without inspecting private internals.
+  class Capabilities
+    # Built-in OSS data-plane feature capabilities.
+    FEATURES = %w[
+      checkpointing
+      delivery_ledger
+      dead_letters
+      replay
+      health
+      metrics
+      routing
+      fanout
+    ].freeze
+
+    attr_reader :config
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    def initialize(config)
+      @config = config
+    end
+
+    # @param config [Mammoth::Configuration] loaded configuration
+    # @return [Hash] JSON-friendly capabilities
+    def self.call(config)
+      new(config).to_h
+    end
+
+    # @return [Hash] JSON-friendly capabilities
+    def to_h
+      {
+        operational_state: operational_state_adapter,
+        destinations: destination_types,
+        runtimes: Runtimes::Registry.names,
+        runtime: runtime_adapter,
+        features: FEATURES
+      }
+    end
+
+    private
+
+    def operational_state_adapter
+      config.dig("operational_state", "adapter") || "sqlite"
+    end
+
+    def runtime_adapter
+      config.dig("runtime", "adapter") || "inline"
+    end
+
+    def destination_types
+      destinations = config.data["destinations"]
+      return [config.dig("webhook", "type") || "webhook"] unless destinations
+
+      destinations.map { |destination| destination.fetch("type", "webhook") }.uniq.sort
+    end
+  end
+end

data/lib/mammoth/cli.rb

@@ -24,9 +24,11 @@ module Mammoth
       "  mammoth status CONFIG",
       "  mammoth start CONFIG",
       "  mammoth deliver-sample CONFIG EVENT_JSON",
-      "  mammoth dead-letters list CONFIG [--status STATUS] [--limit N]",
+      "  mammoth dead-letters list CONFIG [--status STATUS] [--destination NAME] " \
+      "[--failed-after ISO8601] [--failed-before ISO8601] [--limit N]",
       "  mammoth dead-letters show CONFIG ID",
-      "  mammoth dead-letters replay CONFIG [ID ...]",
+      "  mammoth dead-letters replay CONFIG [ID ...] [--destination NAME] [--status STATUS] " \
+      "[--failed-after ISO8601] [--failed-before ISO8601] [--limit N]",
       "  mammoth observability CONFIG"
     ].join("\n")
 
@@ -100,8 +102,7 @@ module Mammoth
     def status
       config = load_config
       store = SQLiteStore.connect(config.dig("sqlite", "path"))
-      Status.call(config, sqlite_store: store)
-      0
+      Commands::StatusCommand.new(config, sqlite_store: store).call
     end
 
     def start

data/lib/mammoth/commands/status_command.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Local command objects shared by CLI and future control-plane agents.
+  module Commands
+    # Reusable local command object for status inspection.
+    class StatusCommand
+      attr_reader :config, :sqlite_store, :output
+
+      # @param config [Mammoth::Configuration] loaded configuration
+      # @param sqlite_store [Mammoth::SQLiteStore, nil] optional operational store
+      # @param output [#puts] output stream
+      def initialize(config, sqlite_store: nil, output: $stdout)
+        @config = config
+        @sqlite_store = sqlite_store
+        @output = output
+      end
+
+      # Execute the status command.
+      #
+      # @return [Integer] process-style status code
+      def call
+        Status.call(config, sqlite_store: sqlite_store, output: output)
+        0
+      end
+    end
+  end
+end