mammoth 0.1.1 → 0.2.0

data/config/mammoth.schema.json

@@ -1,10 +1,9 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
   "id": "https://kanutocd.github.io/mammoth/schema/mammoth.schema.json",
- "title": "Mammoth Configuration",
+  "title": "Mammoth Configuration",
   "type": "object",
   "additionalProperties": false,
-
   "required": [
     "mammoth",
     "postgres",
@@ -14,20 +13,22 @@
     "sqlite",
     "logging"
   ],
-
   "properties": {
     "mammoth": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["name"],
+      "required": [
+        "name"
+      ],
       "properties": {
         "name": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "Human-readable Mammoth instance name used in logs, status output, metrics, and future dashboards. Keep stable per environment."
         }
-      }
+      },
+      "description": "Mammoth instance identity and operator-facing metadata."
     },
-
     "postgres": {
       "type": "object",
       "additionalProperties": false,
@@ -41,28 +42,33 @@
       "properties": {
         "host": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "PostgreSQL host reachable from the Mammoth process or pod."
         },
         "port": {
           "type": "integer",
           "minimum": 1,
-          "maximum": 65535
+          "maximum": 65535,
+          "description": "PostgreSQL TCP port."
         },
         "database": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "Source PostgreSQL database name."
         },
         "username": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "PostgreSQL user used by Mammoth. This user must be able to connect in replication mode and read publication metadata."
         },
         "password_env": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "Name of the environment variable containing the PostgreSQL password. The password itself should not be stored in YAML."
         }
-      }
+      },
+      "description": "PostgreSQL connection settings for the source database."
     },
-
     "replication": {
       "type": "object",
       "additionalProperties": false,
@@ -73,7 +79,8 @@
       "properties": {
         "slot": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "PostgreSQL logical replication slot name. Permanent slots preserve WAL until Mammoth acknowledges progress. Changing this value creates a separate replication position."
         },
         "publications": {
           "type": "array",
@@ -81,25 +88,33 @@
           "items": {
             "type": "string",
             "minLength": 1
-          }
+          },
+          "description": "PostgreSQL publications to subscribe to. Publications define which table changes PostgreSQL sends to Mammoth."
         },
         "start_lsn": {
-          "type": ["string", "null"]
+          "type": [
+            "string",
+            "null"
+          ],
+          "description": "Optional Log Sequence Number for advanced recovery or controlled replay. Leave null or unset for normal checkpoint-based operation."
         },
         "auto_create_slot": {
-          "type": "boolean"
+          "type": "boolean",
+          "description": "When true, Mammoth creates the replication slot if it does not exist. Convenient for development; DBA-managed slots are recommended for production."
         },
         "temporary_slot": {
-          "type": "boolean"
+          "type": "boolean",
+          "description": "When true, use a temporary slot that disappears when the replication session ends. Durable deployments should use false."
         },
         "feedback_interval": {
           "type": "number",
           "minimum": 0,
-          "exclusiveMinimum": true
+          "exclusiveMinimum": true,
+          "description": "Standby feedback interval in seconds. Lower values improve liveness/progress visibility; higher values reduce chatter."
         }
-      }
+      },
+      "description": "Logical replication settings. Mammoth uses one replication slot and one WAL stream; downstream runtime concurrency does not create additional replication connections."
     },
-
     "webhook": {
       "type": "object",
       "additionalProperties": false,
@@ -111,19 +126,69 @@
       "properties": {
         "name": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "Destination name used in logs, dead-letter records, and future metrics."
         },
         "url": {
           "type": "string",
-          "format": "uri"
+          "format": "uri",
+          "description": "Destination URL for delivered CDC payloads. Transaction delivery sends a TransactionEnvelope payload; event delivery sends a single event payload."
         },
         "timeout_seconds": {
           "type": "integer",
-          "minimum": 1
+          "minimum": 1,
+          "description": "HTTP timeout in seconds for webhook delivery attempts."
+        },
+        "headers": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          },
+          "description": "Static HTTP headers added to every webhook request. Do not put secrets here; use header_env for secret-backed values."
+        },
+        "header_env": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string",
+            "minLength": 1
+          },
+          "description": "HTTP headers whose values are read from environment variables. Keys are header names; values are environment variable names."
+        },
+        "signing": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "secret_env"
+          ],
+          "properties": {
+            "algorithm": {
+              "type": "string",
+              "enum": [
+                "hmac_sha256"
+              ],
+              "description": "Webhook signing algorithm. Mammoth signs '<timestamp>.<body>' using HMAC-SHA256."
+            },
+            "secret_env": {
+              "type": "string",
+              "minLength": 1,
+              "description": "Environment variable containing the webhook signing secret."
+            },
+            "signature_header": {
+              "type": "string",
+              "minLength": 1,
+              "description": "Header name used for the generated signature. Defaults to X-Mammoth-Signature."
+            },
+            "timestamp_header": {
+              "type": "string",
+              "minLength": 1,
+              "description": "Header name used for the signing timestamp. Defaults to X-Mammoth-Timestamp."
+            }
+          },
+          "description": "Optional HMAC-SHA256 request signing for webhook receivers."
         }
-      }
+      },
+      "description": "HTTP webhook destination settings."
     },
-
     "retry": {
       "type": "object",
       "additionalProperties": false,
@@ -134,7 +199,8 @@
       "properties": {
         "max_attempts": {
           "type": "integer",
-          "minimum": 1
+          "minimum": 1,
+          "description": "Maximum delivery attempts before moving the event or envelope to the dead-letter store."
         },
         "schedule_seconds": {
           "type": "array",
@@ -142,27 +208,33 @@
           "items": {
             "type": "integer",
             "minimum": 1
-          }
+          },
+          "description": "Progressive retry delay schedule in seconds. If attempts exceed the schedule length, the final value may be reused."
         }
-      }
+      },
+      "description": "Retry policy for failed delivery attempts before dead-lettering."
     },
-
     "sqlite": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["path"],
+      "required": [
+        "path"
+      ],
       "properties": {
         "path": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "description": "Filesystem path to Mammoth SQLite operational database. In Kubernetes, back this path with persistent storage."
         }
-      }
+      },
+      "description": "SQLite operational store settings for checkpoints, attempts, and dead-letter records."
     },
-
     "logging": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["level"],
+      "required": [
+        "level"
+      ],
       "properties": {
         "level": {
           "type": "string",
@@ -171,9 +243,83 @@
             "info",
             "warn",
             "error"
-          ]
+          ],
+          "description": "Log verbosity. Use debug for diagnostics, info for normal operation, and warn/error for quieter production output."
+        }
+      },
+      "description": "Logging configuration."
+    },
+    "delivery": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "unit": {
+          "type": "string",
+          "enum": [
+            "event",
+            "transaction"
+          ],
+          "description": "Delivery granularity. event delivers each change independently; transaction delivers a complete TransactionEnvelope and is the safer default for checkpoint correctness."
+        },
+        "ordering": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "scope": {
+              "type": "string",
+              "enum": [
+                "global",
+                "transaction",
+                "relation",
+                "primary_key",
+                "none"
+              ],
+              "description": "Ordering scope. global preserves complete WAL order; transaction preserves commit order; relation preserves per-table order; primary_key preserves per-row order; none is best-effort only."
+            }
+          },
+          "description": "Ordering policy consumed by the delivery runtime."
+        }
+      },
+      "description": "Delivery semantics for CDC payloads."
+    },
+    "runtime": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "adapter": {
+          "type": "string",
+          "enum": [
+            "inline",
+            "concurrent"
+          ],
+          "description": "Delivery runtime adapter. inline executes synchronously; concurrent uses CDC::Concurrent::ProcessorPool for I/O-heavy delivery work."
+        },
+        "concurrency": {
+          "type": "integer",
+          "minimum": 1,
+          "description": "Number of downstream delivery workers. Does not create extra PostgreSQL replication slots or connections."
+        },
+        "batch_size": {
+          "type": "integer",
+          "minimum": 1,
+          "description": "Number of work units submitted to the concurrent runtime together. Values greater than 1 allow ordering-insensitive deliveries to overlap more visibly."
+        },
+        "preserve_order": {
+          "type": "boolean",
+          "description": "When true, preserve configured delivery ordering while using the runtime. Disable only for idempotent ordering-insensitive destinations."
+        },
+        "timeout_seconds": {
+          "type": [
+            "number",
+            "null"
+          ],
+          "minimum": 0,
+          "exclusiveMinimum": true,
+          "description": "Optional runtime-level timeout in seconds. Leave null to rely on destination-specific timeouts."
         }
-      }
+      },
+      "description": "Downstream delivery runtime settings. Runtime concurrency affects webhook/destination delivery only, not PostgreSQL replication stream count."
     }
-  }
-}
+  },
+  "description": "Mammoth relay configuration. Mammoth consumes PostgreSQL logical replication changes and delivers them to downstream destinations while preserving checkpoint, retry, and dead-letter semantics."
+}

data/lib/mammoth/application.rb

@@ -10,7 +10,7 @@ module Mammoth
   # injected CDC work source rather than owning upstream CDC source-adapter
   # lifecycle decisions.
   class Application
-    attr_reader :config, :sqlite_store, :consumer, :delivery_worker
+    attr_reader :config, :sqlite_store, :consumer, :delivery_worker, :checkpoint_store
 
     # @param config [Mammoth::Configuration] loaded configuration
     # @param source [#each, nil] injectable event source for tests and demos
@@ -19,32 +19,111 @@ module Mammoth
     def initialize(config, source: nil, sink: nil, sleeper: Kernel.method(:sleep))
       @config = config
       @sqlite_store = SQLiteStore.connect(config.dig("sqlite", "path")).bootstrap!
-      @consumer = ReplicationConsumer.new(source: source)
+      @checkpoint_store = CheckpointStore.new(sqlite_store)
+      @consumer = ReplicationConsumer.new(source: source || build_source, delivery_unit: delivery_unit)
       @delivery_worker = build_delivery_worker(sink: sink || WebhookSink.from_config(config), sleeper: sleeper)
     end
 
-    # Start the application runtime and deliver consumed events.
+    # Start the application runtime and deliver consumed CDC work.
     #
-    # @return [Integer] number of processed events
+    # @return [Integer] number of processed work units
     def start
+      runtime = build_runtime
       processed = 0
-      consumer.start do |event|
-        delivery_worker.deliver(event)
-        processed += 1
+      batch = [nil].compact
+
+      consumer.start do |work|
+        if runtime_batching?(runtime)
+          batch << work
+          next unless batch.size >= runtime_batch_size
+
+          processed += process_batch(runtime, batch)
+          batch = []
+        else
+          process_work(runtime, work)
+          processed += 1
+        end
       end
+
+      processed += process_batch(runtime, batch) if runtime_batching?(runtime) && batch.any?
       processed
+    ensure
+      runtime.shutdown if runtime.respond_to?(:shutdown)
     end
 
     private
 
+    def process_work(runtime, work)
+      if runtime
+        runtime.process_many([work])
+      elsif transaction_delivery?
+        delivery_worker.deliver_transaction(work)
+      else
+        delivery_worker.deliver(work)
+      end
+    end
+
+    def process_batch(runtime, batch)
+      runtime.process_many(batch)
+      batch.size
+    end
+
+    def runtime_batching?(runtime)
+      runtime && runtime_batch_size > 1
+    end
+
+    def build_runtime
+      return unless runtime_adapter == "concurrent"
+
+      ConcurrentDeliveryRuntime.new(
+        processor: DeliveryProcessor.new(delivery_worker:, delivery_unit: delivery_unit),
+        concurrency: runtime_concurrency,
+        timeout: runtime_timeout,
+        preserve_order: runtime_preserve_order?
+      )
+    end
+
+    def build_source
+      Sources::Postgres.new(config, checkpoint_store: checkpoint_store)
+    end
+
     def build_delivery_worker(sink:, sleeper:)
       DeliveryWorker.from_config(
         config,
         sink: sink,
-        checkpoint_store: CheckpointStore.new(sqlite_store),
+        checkpoint_store: checkpoint_store,
         dead_letter_store: DeadLetterStore.new(sqlite_store),
         sleeper: sleeper
       )
     end
+
+    def transaction_delivery?
+      delivery_unit == :transaction
+    end
+
+    def delivery_unit
+      (config.dig("delivery", "unit") || "event").to_sym
+    end
+
+    def runtime_adapter
+      config.dig("runtime", "adapter") || "inline"
+    end
+
+    def runtime_concurrency
+      config.dig("runtime", "concurrency") || 1
+    end
+
+    def runtime_timeout
+      config.dig("runtime", "timeout_seconds")
+    end
+
+    def runtime_batch_size
+      config.dig("runtime", "batch_size") || 1
+    end
+
+    def runtime_preserve_order?
+      value = config.dig("runtime", "preserve_order")
+      value.nil? || value
+    end
   end
 end

data/lib/mammoth/cli.rb

@@ -90,7 +90,7 @@ module Mammoth
 
     def start
       config = load_config
-      processed = Application.new(config, source: Sources::Postgres.new(config)).start
+      processed = Application.new(config).start
       puts "Processed events: #{processed}"
       0
     end

data/lib/mammoth/concurrent_delivery_runtime.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Mammoth
+  # Delivery runtime backed by CDC::Concurrent::ProcessorPool.
+  #
+  # Mammoth keeps a single upstream replication stream and delegates downstream
+  # I/O fan-out to cdc-concurrent. This class is intentionally small so the
+  # runtime boundary remains easy to test and replace.
+  class ConcurrentDeliveryRuntime
+    attr_reader :processor, :concurrency, :timeout, :preserve_order, :pool
+
+    # @param processor [#process] delivery processor
+    # @param concurrency [Integer] number of concurrent delivery workers
+    # @param timeout [Numeric, nil] optional per-item timeout
+    # @param preserve_order [Boolean] preserve output order when supported
+    def initialize(processor:, concurrency:, timeout:, preserve_order:)
+      @processor = processor
+      @concurrency = concurrency
+      @timeout = timeout
+      @preserve_order = preserve_order
+      @pool = build_pool
+    end
+
+    # Process many work items through cdc-concurrent.
+    #
+    # @param items [Array<Object>] CDC work units
+    # @return [Array<Object>] processor results
+    def process_many(items)
+      return [] if items.empty?
+
+      pool.process_many(items)
+    end
+
+    # Shutdown the underlying runtime when supported.
+    #
+    # @return [nil]
+    def shutdown
+      pool.shutdown if pool.respond_to?(:shutdown)
+      nil
+    end
+
+    private
+
+    def build_pool
+      require "cdc/concurrent"
+      CDC::Concurrent::ProcessorPool.new(processor:, concurrency:, timeout:, preserve_order:)
+    rescue LoadError => e
+      raise ConfigurationError,
+            "runtime.adapter=concurrent requires cdc-concurrent. Add `gem \"cdc-concurrent\"`. Original error: #{e.message}"
+    end
+  end
+end

data/lib/mammoth/dead_letter_store.rb

@@ -20,9 +20,9 @@ module Mammoth
     # @param error [Exception, nil] delivery failure
     # @param retry_count [Integer] number of delivery attempts
     # @return [Integer] inserted dead letter id
-    def write(event:, destination_name:, error: nil, retry_count: 0) # rubocop:disable Metrics/MethodLength
+    def write(event:, destination_name:, error: nil, retry_count: 0, serializer: EventSerializer) # rubocop:disable Metrics/MethodLength
       now = Time.now.utc.iso8601
-      payload = EventSerializer.call(event)
+      payload = serializer.call(event)
       database.execute(
         <<~SQL,
           INSERT INTO dead_letters(
@@ -46,7 +46,7 @@ module Mammoth
           payload.fetch("event_id"),
           payload.fetch("source"),
           destination_name,
-          payload.fetch("operation"),
+          payload["operation"] || payload["type"],
           payload["namespace"],
           payload["entity"],
           payload["source_position"],

data/lib/mammoth/delivered_envelope_store.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Mammoth
+  # SQLite-backed ledger of downstream deliveries.
+  #
+  # The PostgreSQL replication boundary is at-least-once. Mammoth therefore
+  # keeps a small delivery ledger so a transaction replayed by the upstream
+  # replication source after restart does not have to be delivered downstream again.
+  class DeliveredEnvelopeStore
+    SCHEMA = <<~SQL
+      CREATE TABLE IF NOT EXISTS delivered_envelopes (
+        id INTEGER PRIMARY KEY,
+        idempotency_key TEXT NOT NULL,
+        source_name TEXT NOT NULL,
+        slot_name TEXT NOT NULL,
+        destination_name TEXT NOT NULL,
+        delivery_unit TEXT NOT NULL,
+        transaction_id TEXT,
+        source_position TEXT,
+        delivered_at TEXT NOT NULL,
+
+        UNIQUE (idempotency_key)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source
+      ON delivered_envelopes(source_name, slot_name);
+
+      CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_destination
+      ON delivered_envelopes(destination_name);
+
+      CREATE INDEX IF NOT EXISTS idx_delivered_envelopes_source_position
+      ON delivered_envelopes(source_position);
+    SQL
+
+    attr_reader :sqlite_store
+
+    def initialize(sqlite_store)
+      @sqlite_store = sqlite_store
+      ensure_schema!
+    end
+
+    def delivered?(idempotency_key)
+      !database.execute(
+        "SELECT 1 FROM delivered_envelopes WHERE idempotency_key = ? LIMIT 1",
+        [idempotency_key]
+      ).empty?
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def record!(idempotency_key:, source_name:, slot_name:, destination_name:, delivery_unit:, transaction_id:,
+                source_position:)
+      database.execute(
+        <<~SQL,
+          INSERT OR IGNORE INTO delivered_envelopes(
+            idempotency_key, source_name, slot_name, destination_name, delivery_unit,
+            transaction_id, source_position, delivered_at
+          )
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        SQL
+        [
+          idempotency_key,
+          source_name,
+          slot_name,
+          destination_name,
+          delivery_unit,
+          transaction_id,
+          source_position,
+          Time.now.utc.iso8601
+        ]
+      )
+      database.get_first_row(
+        "SELECT * FROM delivered_envelopes WHERE idempotency_key = ? LIMIT 1",
+        [idempotency_key]
+      )
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def all
+      database.execute("SELECT * FROM delivered_envelopes ORDER BY id")
+    end
+
+    def count
+      database.get_first_value("SELECT COUNT(*) FROM delivered_envelopes")
+    end
+
+    private
+
+    def ensure_schema!
+      database.execute_batch(SCHEMA)
+    end
+
+    def database
+      sqlite_store.bootstrap!.database
+    end
+  end
+end