mammoth 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +38 -0
  3. data/README.md +25 -4
  4. data/config/mammoth.schema.json +1 -1
  5. data/lib/mammoth/application.rb +20 -40
  6. data/lib/mammoth/cli.rb +1 -13
  7. data/lib/mammoth/commands/bootstrap_command.rb +11 -7
  8. data/lib/mammoth/commands/dead_letters_command.rb +5 -3
  9. data/lib/mammoth/commands/deliver_sample_command.rb +1 -1
  10. data/lib/mammoth/commands/status_command.rb +5 -5
  11. data/lib/mammoth/concurrent_delivery_runtime.rb +15 -3
  12. data/lib/mammoth/dead_letter_commands.rb +16 -25
  13. data/lib/mammoth/dead_letter_store.rb +1 -1
  14. data/lib/mammoth/delivery_processor.rb +46 -7
  15. data/lib/mammoth/delivery_worker.rb +10 -8
  16. data/lib/mammoth/dispatch_metrics.rb +55 -0
  17. data/lib/mammoth/event_serializer.rb +23 -12
  18. data/lib/mammoth/fanout_delivery_worker.rb +4 -4
  19. data/lib/mammoth/metrics_observer.rb +52 -0
  20. data/lib/mammoth/observability_metrics.rb +69 -0
  21. data/lib/mammoth/observability_server.rb +6 -6
  22. data/lib/mammoth/observability_snapshot.rb +59 -60
  23. data/lib/mammoth/operational_state/adapter.rb +16 -0
  24. data/lib/mammoth/operational_state/registry.rb +8 -0
  25. data/lib/mammoth/operational_state/sqlite_adapter.rb +18 -2
  26. data/lib/mammoth/persisted_payload_deserializer.rb +85 -0
  27. data/lib/mammoth/replication_consumer.rb +15 -47
  28. data/lib/mammoth/runtimes/batching_runtime.rb +59 -0
  29. data/lib/mammoth/runtimes/concurrent_adapter.rb +4 -2
  30. data/lib/mammoth/runtimes/inline_adapter.rb +19 -5
  31. data/lib/mammoth/runtimes/registry.rb +3 -2
  32. data/lib/mammoth/sources/postgres.rb +10 -1
  33. data/lib/mammoth/status.rb +22 -18
  34. data/lib/mammoth/transaction_envelope_serializer.rb +13 -15
  35. data/lib/mammoth/version.rb +1 -1
  36. data/lib/mammoth/webhook_sink.rb +2 -2
  37. data/lib/mammoth.rb +8 -1
  38. metadata +6 -1
@@ -17,7 +17,7 @@ module Mammoth
17
17
  # @param sink [#deliver] destination sink
18
18
  # @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
19
19
  # @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
20
- # @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore, nil] downstream delivery ledger
20
+ # @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore] downstream delivery ledger
21
21
  # @param source_name [String] logical source name
22
22
  # @param slot_name [String] replication slot name
23
23
  # @param publication_name [String] publication name
@@ -26,13 +26,13 @@ module Mammoth
26
26
  # @param sleeper [#call] sleep strategy, injectable for tests
27
27
  # @param route_filter [Mammoth::RouteFilter, nil] optional destination route matcher
28
28
  # @param enabled [Boolean] whether this destination accepts new deliveries
29
- def initialize(sink:, checkpoint_store:, dead_letter_store:, source_name:, slot_name:, publication_name:,
30
- max_attempts:, retry_schedule:, delivered_envelope_store: nil, sleeper: Kernel.method(:sleep),
29
+ def initialize(sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:, source_name:, slot_name:,
30
+ publication_name:, max_attempts:, retry_schedule:, sleeper: Kernel.method(:sleep),
31
31
  route_filter: nil, enabled: true)
32
32
  @sink = sink
33
33
  @checkpoint_store = checkpoint_store
34
34
  @dead_letter_store = dead_letter_store
35
- @delivered_envelope_store = delivered_envelope_store || DeliveredEnvelopeStore.new(checkpoint_store.sqlite_store)
35
+ @delivered_envelope_store = delivered_envelope_store
36
36
  @source_name = source_name
37
37
  @slot_name = slot_name
38
38
  @publication_name = publication_name
@@ -49,14 +49,16 @@ module Mammoth
49
49
  # @param sink [#deliver] destination sink
50
50
  # @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
51
51
  # @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
52
+ # @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore] downstream delivery ledger
52
53
  # @param sleeper [#call] sleep strategy
53
54
  # @return [Mammoth::DeliveryWorker]
54
- def self.from_config(config, sink:, checkpoint_store:, dead_letter_store:, sleeper: Kernel.method(:sleep),
55
- delivery_policy: {})
55
+ def self.from_config(config, sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:,
56
+ sleeper: Kernel.method(:sleep), delivery_policy: {})
56
57
  new(
57
58
  sink: sink,
58
59
  checkpoint_store: checkpoint_store,
59
60
  dead_letter_store: dead_letter_store,
61
+ delivered_envelope_store: delivered_envelope_store,
60
62
  source_name: config.dig("mammoth", "name"),
61
63
  slot_name: config.dig("replication", "slot"),
62
64
  publication_name: Array(config.dig("replication", "publications")).join(","),
@@ -70,7 +72,7 @@ module Mammoth
70
72
 
71
73
  # Deliver a transaction envelope with retry, checkpoint, and DLQ handling.
72
74
  #
73
- # @param envelope [#events, #transaction_id] CDC transaction envelope
75
+ # @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
74
76
  # @return [Hash] delivery summary
75
77
  def deliver_transaction(envelope)
76
78
  deliver_work(envelope, serializer: TransactionEnvelopeSerializer, delivery_method: :deliver_transaction)
@@ -78,7 +80,7 @@ module Mammoth
78
80
 
79
81
  # Deliver an event with retry, checkpoint, and DLQ handling.
80
82
  #
81
- # @param event [Hash, #to_h] normalized event
83
+ # @param event [CDC::Core::ChangeEvent] normalized event
82
84
  # @return [Hash] delivery summary
83
85
  def deliver(event)
84
86
  deliver_work(event, serializer: EventSerializer, delivery_method: :deliver)
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mammoth
4
+ # Thread-safe in-process counters populated by Mammoth's CDC core observer.
5
+ #
6
+ # The registry deliberately stores CDC core metric names and tags. Rendering
7
+ # those counters for a specific backend remains an observability concern.
8
+ class DispatchMetrics
9
+ # Create an empty dispatch metric registry.
10
+ def initialize
11
+ @mutex = Mutex.new
12
+ @counters = Hash.new(0)
13
+ end
14
+
15
+ # Process-wide dispatch metrics used by the default observer and snapshot.
16
+ INSTANCE = new
17
+
18
+ # Increment a canonical CDC core metric.
19
+ #
20
+ # @param name [String] canonical CDC core metric name
21
+ # @param tags [Hash] canonical metric tags
22
+ # @return [Integer] updated counter value
23
+ def increment(name, tags = {})
24
+ key = [name.to_s.freeze, normalized_tags(tags)]
25
+ @mutex.synchronize { @counters[key] += 1 }
26
+ end
27
+
28
+ # Return an immutable point-in-time copy of all counters.
29
+ #
30
+ # @return [Array<Hash>] metric entries with name, tags, and value
31
+ def snapshot
32
+ @mutex.synchronize do
33
+ @counters.map do |(name, tags), value|
34
+ { name: name, tags: tags.to_h, value: value }
35
+ end
36
+ end
37
+ end
38
+
39
+ # Clear all counters.
40
+ #
41
+ # Intended for process lifecycle management and isolated tests.
42
+ #
43
+ # @return [self] cleared registry
44
+ def reset!
45
+ @mutex.synchronize { @counters.clear }
46
+ self
47
+ end
48
+
49
+ private
50
+
51
+ def normalized_tags(tags)
52
+ tags.to_h.transform_keys(&:to_s).sort_by(&:first).freeze
53
+ end
54
+ end
55
+ end
@@ -15,17 +15,19 @@ module Mammoth
15
15
  # Default source label used in serialized webhook payloads.
16
16
  DEFAULT_SOURCE = "postgresql"
17
17
 
18
- # Serialize an event-like object into a webhook-ready Hash.
18
+ # Serialize a core change event into a webhook-ready Hash.
19
19
  #
20
- # @param event [Hash, #to_h] normalized CDC event
20
+ # @param event [CDC::Core::ChangeEvent] normalized CDC event
21
21
  # @return [Hash] webhook payload
22
22
  def self.call(event)
23
23
  new(event).call
24
24
  end
25
25
 
26
- # @param event [Hash, #to_h] normalized CDC event
26
+ # @param event [CDC::Core::ChangeEvent] normalized CDC event
27
27
  def initialize(event)
28
- @event = event.respond_to?(:to_h) ? event.to_h : event
28
+ raise ArgumentError, "event must be a CDC::Core::ChangeEvent" unless event.is_a?(CDC::Core::ChangeEvent)
29
+
30
+ @event = event.to_h
29
31
  end
30
32
 
31
33
  # Return the webhook payload.
@@ -33,19 +35,20 @@ module Mammoth
33
35
  # @return [Hash] webhook payload
34
36
  def call
35
37
  event_hash = stringify_keys(@event)
38
+ metadata = stringify_keys(event_hash["metadata"] || {})
36
39
  {
37
- "event_id" => event_hash["event_id"] || SecureRandom.uuid,
38
- "source" => event_hash["source"] || DEFAULT_SOURCE,
40
+ "event_id" => event_id(metadata),
41
+ "source" => source(metadata),
39
42
  "operation" => normalize_operation(event_hash.fetch("operation")),
40
- "namespace" => event_hash["namespace"] || event_hash["schema"],
41
- "entity" => event_hash["entity"] || event_hash["table"],
42
- "identity" => event_hash["identity"] || event_hash["primary_key"],
43
- "source_position" => event_hash["source_position"] || event_hash["commit_lsn"],
43
+ "namespace" => event_hash["schema"],
44
+ "entity" => event_hash["table"],
45
+ "identity" => event_hash["primary_key"],
46
+ "source_position" => event_hash["commit_lsn"],
44
47
  "transaction_id" => event_hash["transaction_id"],
45
48
  "occurred_at" => occurred_at(event_hash),
46
49
  "data" => event_data(event_hash),
47
- "changes" => event_hash["changes"] || [],
48
- "metadata" => event_hash["metadata"] || {}
50
+ "changes" => metadata["changes"] || [],
51
+ "metadata" => metadata
49
52
  }
50
53
  end
51
54
 
@@ -66,6 +69,14 @@ module Mammoth
66
69
  operation.to_s
67
70
  end
68
71
 
72
+ def event_id(metadata)
73
+ metadata["event_id"] || SecureRandom.uuid
74
+ end
75
+
76
+ def source(metadata)
77
+ metadata["source"] || DEFAULT_SOURCE
78
+ end
79
+
69
80
  def occurred_at(event_hash)
70
81
  value = event_hash["occurred_at"]
71
82
  return value.iso8601 if value.respond_to?(:iso8601)
@@ -14,7 +14,7 @@ module Mammoth
14
14
 
15
15
  # Deliver an event to every configured destination.
16
16
  #
17
- # @param event [Hash, #to_h] normalized event
17
+ # @param event [CDC::Core::ChangeEvent] normalized event
18
18
  # @return [Hash] aggregate fanout summary
19
19
  def deliver(event)
20
20
  fanout(:deliver, event)
@@ -22,7 +22,7 @@ module Mammoth
22
22
 
23
23
  # Deliver a transaction envelope to every configured destination.
24
24
  #
25
- # @param envelope [#events, #transaction_id] CDC transaction envelope
25
+ # @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
26
26
  # @return [Hash] aggregate fanout summary
27
27
  def deliver_transaction(envelope)
28
28
  fanout(:deliver_transaction, envelope)
@@ -31,7 +31,7 @@ module Mammoth
31
31
  # Deliver an event to one configured destination.
32
32
  #
33
33
  # @param destination_name [String] destination name
34
- # @param event [Hash, #to_h] normalized event
34
+ # @param event [CDC::Core::ChangeEvent] normalized event
35
35
  # @return [Hash] destination delivery summary
36
36
  def deliver_to(destination_name, event)
37
37
  worker_for(destination_name).deliver(event)
@@ -40,7 +40,7 @@ module Mammoth
40
40
  # Deliver a transaction envelope to one configured destination.
41
41
  #
42
42
  # @param destination_name [String] destination name
43
- # @param envelope [#events, #transaction_id] CDC transaction envelope
43
+ # @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
44
44
  # @return [Hash] destination delivery summary
45
45
  def deliver_transaction_to(destination_name, envelope)
46
46
  worker_for(destination_name).deliver_transaction(envelope)
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mammoth
4
+ # CDC core observer that records Mammoth dispatch counters.
5
+ class MetricsObserver < CDC::Core::Observer
6
+ attr_reader :metrics
7
+
8
+ # @param metrics [Mammoth::DispatchMetrics] dispatch counter registry
9
+ def initialize(metrics: DispatchMetrics::INSTANCE)
10
+ super()
11
+ @metrics = metrics
12
+ end
13
+
14
+ # Record a dispatch attempt.
15
+ #
16
+ # @param event [Object] CDC work item
17
+ # @return [Integer] updated counter
18
+ def dispatch_started(event)
19
+ record(self.class.started_metric_name, event)
20
+ end
21
+
22
+ # Record a successful dispatch.
23
+ #
24
+ # @param result [CDC::Core::ProcessorResult] processor result
25
+ # @return [Integer] updated counter
26
+ def dispatch_succeeded(result)
27
+ record(self.class.succeeded_metric_name, result)
28
+ end
29
+
30
+ # Record a failed dispatch.
31
+ #
32
+ # @param result [CDC::Core::ProcessorResult] processor result
33
+ # @return [Integer] updated counter
34
+ def dispatch_failed(result)
35
+ record(self.class.failed_metric_name, result)
36
+ end
37
+
38
+ # Record a skipped dispatch.
39
+ #
40
+ # @param result [CDC::Core::ProcessorResult] processor result
41
+ # @return [Integer] updated counter
42
+ def dispatch_skipped(result)
43
+ record(self.class.skipped_metric_name, result)
44
+ end
45
+
46
+ private
47
+
48
+ def record(name, payload)
49
+ metrics.increment(name, CDC::Core::Observer.metric_tags(payload))
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Mammoth
4
+ # Prometheus formatting helpers for operational and dispatch metrics.
5
+ module ObservabilityMetrics
6
+ # Maps canonical CDC core metric names to Mammoth's Prometheus counters.
7
+ DISPATCH_METRIC_NAMES = {
8
+ CDC::Core::Observer.started_metric_name => "mammoth_dispatch_started_total",
9
+ CDC::Core::Observer.succeeded_metric_name => "mammoth_dispatch_succeeded_total",
10
+ CDC::Core::Observer.failed_metric_name => "mammoth_dispatch_failed_total",
11
+ CDC::Core::Observer.skipped_metric_name => "mammoth_dispatch_skipped_total"
12
+ }.freeze
13
+
14
+ private
15
+
16
+ def metric_headers
17
+ operational_metric_headers + dispatch_metric_headers
18
+ end
19
+
20
+ def operational_metric_headers
21
+ [
22
+ "# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
23
+ "# TYPE mammoth_up gauge",
24
+ "# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
25
+ "# TYPE mammoth_checkpoints_total gauge",
26
+ "# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
27
+ "# TYPE mammoth_dead_letters_total gauge",
28
+ "# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
29
+ "# TYPE mammoth_dead_letters_pending_total gauge",
30
+ "# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
31
+ "# TYPE mammoth_dead_letters_resolved_total gauge",
32
+ "# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
33
+ "# TYPE mammoth_dead_letters_ignored_total gauge",
34
+ "# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
35
+ "# TYPE mammoth_delivered_envelopes_total gauge"
36
+ ]
37
+ end
38
+
39
+ def dispatch_metric_headers
40
+ [
41
+ "# HELP mammoth_dispatch_started_total Number of CDC work items submitted for delivery.",
42
+ "# TYPE mammoth_dispatch_started_total counter",
43
+ "# HELP mammoth_dispatch_succeeded_total Number of successful CDC delivery results.",
44
+ "# TYPE mammoth_dispatch_succeeded_total counter",
45
+ "# HELP mammoth_dispatch_failed_total Number of failed CDC delivery results.",
46
+ "# TYPE mammoth_dispatch_failed_total counter",
47
+ "# HELP mammoth_dispatch_skipped_total Number of skipped CDC delivery results.",
48
+ "# TYPE mammoth_dispatch_skipped_total counter"
49
+ ]
50
+ end
51
+
52
+ def dispatch_metric_lines
53
+ dispatch_metrics.snapshot.sort_by { |entry| [entry.fetch(:name), entry.fetch(:tags).to_a] }.filter_map do |entry|
54
+ name = DISPATCH_METRIC_NAMES[entry.fetch(:name)]
55
+ metric_line(name, entry.fetch(:value), labels: entry.fetch(:tags)) if name
56
+ end
57
+ end
58
+
59
+ def metric_line(name, value, destination: nil, labels: {})
60
+ labels = { "mammoth_name" => mammoth_name }.merge(labels.transform_keys(&:to_s))
61
+ labels["destination"] = destination if destination
62
+ %(#{name}{#{labels.map { |label, label_value| %(#{label}="#{escape_label(label_value)}") }.join(",")}} #{Integer(value)})
63
+ end
64
+
65
+ def escape_label(value)
66
+ value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
67
+ end
68
+ end
69
+ end
@@ -8,25 +8,25 @@ module Mammoth
8
8
  #
9
9
  # The server is intentionally independent from the replication loop. Operators
10
10
  # may run it as a sidecar-like process or in a separate process that points at
11
- # the same SQLite operational database.
11
+ # the same configured operational-state backend.
12
12
  class ObservabilityServer
13
13
  # Default bind host for the observability server.
14
14
  DEFAULT_HOST = "0.0.0.0"
15
15
  # Default TCP port for the observability server.
16
16
  DEFAULT_PORT = 9393
17
17
 
18
- attr_reader :config, :host, :port, :sqlite_store, :logger, :server
18
+ attr_reader :config, :host, :port, :state_adapter, :logger, :server
19
19
 
20
20
  # @param config [Mammoth::Configuration] loaded configuration
21
21
  # @param host [String, nil] bind host override
22
22
  # @param port [Integer, nil] bind port override
23
- # @param sqlite_store [Mammoth::SQLiteStore, nil] optional operational store
23
+ # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
24
24
  # @param logger [WEBrick::Log, nil] optional WEBrick logger
25
- def initialize(config, host: nil, port: nil, sqlite_store: nil, logger: nil)
25
+ def initialize(config, host: nil, port: nil, state_adapter: nil, logger: nil)
26
26
  @config = config
27
27
  @host = host || config.dig("observability", "host") || DEFAULT_HOST
28
28
  @port = port || config.dig("observability", "port") || DEFAULT_PORT
29
- @sqlite_store = sqlite_store
29
+ @state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
30
30
  @logger = logger || WEBrick::Log.new($stderr, WEBrick::Log::WARN)
31
31
  @server = build_server
32
32
  mount_endpoints
@@ -81,7 +81,7 @@ module Mammoth
81
81
  end
82
82
 
83
83
  def snapshot
84
- ObservabilitySnapshot.new(config, sqlite_store: sqlite_store)
84
+ ObservabilitySnapshot.new(config, state_adapter: state_adapter)
85
85
  end
86
86
  end
87
87
  end
@@ -9,17 +9,21 @@ module Mammoth
9
9
  # ObservabilitySnapshot is intentionally read-only. It does not start the
10
10
  # relay, mutate checkpoints, replay dead letters, or inspect PostgreSQL. The
11
11
  # health and metrics endpoints use this object to expose Mammoth process and
12
- # SQLite operational-state status in a predictable format.
12
+ # operational-state adapter status in a predictable format.
13
13
  class ObservabilitySnapshot
14
- attr_reader :config, :sqlite_store, :clock
14
+ include ObservabilityMetrics
15
+
16
+ attr_reader :config, :state_adapter, :clock, :dispatch_metrics
15
17
 
16
18
  # @param config [Mammoth::Configuration] loaded configuration
17
- # @param sqlite_store [Mammoth::SQLiteStore, nil] optional operational store
19
+ # @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
18
20
  # @param clock [#call] time source returning a Time-like object
19
- def initialize(config, sqlite_store: nil, clock: -> { Time.now.utc })
21
+ # @param dispatch_metrics [Mammoth::DispatchMetrics] dispatch counter registry
22
+ def initialize(config, state_adapter: nil, clock: -> { Time.now.utc }, dispatch_metrics: DispatchMetrics::INSTANCE)
20
23
  @config = config
21
- @sqlite_store = sqlite_store
24
+ @state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
22
25
  @clock = clock
26
+ @dispatch_metrics = dispatch_metrics
23
27
  end
24
28
 
25
29
  # Build a liveness response.
@@ -39,50 +43,55 @@ module Mammoth
39
43
  #
40
44
  # @return [Hash] readiness payload
41
45
  def readiness
42
- store = operational_store
43
- store.bootstrap!
46
+ return unready_payload unless state_adapter.ready?
44
47
 
45
48
  {
46
49
  status: "ready",
47
50
  service: "mammoth",
48
51
  name: mammoth_name,
49
- sqlite: "ok",
50
- tables: store.tables,
51
- checked_at: checked_at
52
- }
53
- rescue Mammoth::Error, SQLite3::Exception => e
54
- {
55
- status: "unready",
56
- service: "mammoth",
57
- name: mammoth_name,
58
- sqlite: "error",
59
- error_class: e.class.name,
60
- error_message: e.message,
52
+ operational_state: "ok",
53
+ adapter: state_summary.fetch(:adapter),
54
+ summary: state_summary,
61
55
  checked_at: checked_at
62
56
  }
57
+ rescue Mammoth::Error => e
58
+ adapter_error_payload(e)
63
59
  end
64
60
 
65
61
  # Build a Prometheus text exposition document.
66
62
  #
67
63
  # @return [String] Prometheus metrics text
68
64
  def prometheus
69
- store = operational_store.bootstrap!
70
- checkpoint_store = CheckpointStore.new(store)
71
- dead_letter_store = DeadLetterStore.new(store)
72
- delivered_store = DeliveredEnvelopeStore.new(store)
65
+ return down_metrics unless state_adapter.ready?
73
66
 
74
67
  lines = metric_headers + aggregate_metric_lines(
75
- checkpoint_store: checkpoint_store,
76
- dead_letter_store: dead_letter_store,
77
- delivered_store: delivered_store
78
- ) + destination_metric_lines(dead_letter_store:, delivered_store:)
68
+ checkpoint_store: state_adapter.checkpoint_store,
69
+ dead_letter_store: state_adapter.dead_letter_store,
70
+ delivered_store: state_adapter.delivered_envelope_store
71
+ ) + destination_metric_lines(
72
+ dead_letter_store: state_adapter.dead_letter_store,
73
+ delivered_store: state_adapter.delivered_envelope_store
74
+ ) + dispatch_metric_lines
79
75
  "#{lines.join("\n")}\n"
80
- rescue Mammoth::Error, SQLite3::Exception
81
- "#{(metric_headers + [metric_line("mammoth_up", 0)]).join("\n")}\n"
76
+ rescue Mammoth::Error
77
+ down_metrics
82
78
  end
83
79
 
84
80
  private
85
81
 
82
+ def adapter_error_payload(error)
83
+ {
84
+ status: "unready",
85
+ service: "mammoth",
86
+ name: mammoth_name,
87
+ operational_state: "error",
88
+ adapter: configured_adapter_name,
89
+ error_class: error.class.name,
90
+ error_message: error.message,
91
+ checked_at: checked_at
92
+ }
93
+ end
94
+
86
95
  def aggregate_metric_lines(checkpoint_store:, dead_letter_store:, delivered_store:)
87
96
  [
88
97
  metric_line("mammoth_up", 1),
@@ -112,8 +121,27 @@ module Mammoth
112
121
  end
113
122
  end
114
123
 
115
- def operational_store
116
- sqlite_store || SQLiteStore.connect(config.dig("sqlite", "path"))
124
+ def unready_payload
125
+ {
126
+ status: "unready",
127
+ service: "mammoth",
128
+ name: mammoth_name,
129
+ operational_state: "error",
130
+ adapter: configured_adapter_name,
131
+ checked_at: checked_at
132
+ }
133
+ end
134
+
135
+ def down_metrics
136
+ "#{(metric_headers + [metric_line("mammoth_up", 0)] + dispatch_metric_lines).join("\n")}\n"
137
+ end
138
+
139
+ def state_summary
140
+ @state_summary ||= state_adapter.summary
141
+ end
142
+
143
+ def configured_adapter_name
144
+ config.dig("operational_state", "adapter") || "sqlite"
117
145
  end
118
146
 
119
147
  def mammoth_name
@@ -130,34 +158,5 @@ module Mammoth
130
158
  def checked_at
131
159
  clock.call.utc.iso8601
132
160
  end
133
-
134
- def metric_headers
135
- [
136
- "# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
137
- "# TYPE mammoth_up gauge",
138
- "# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
139
- "# TYPE mammoth_checkpoints_total gauge",
140
- "# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
141
- "# TYPE mammoth_dead_letters_total gauge",
142
- "# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
143
- "# TYPE mammoth_dead_letters_pending_total gauge",
144
- "# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
145
- "# TYPE mammoth_dead_letters_resolved_total gauge",
146
- "# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
147
- "# TYPE mammoth_dead_letters_ignored_total gauge",
148
- "# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
149
- "# TYPE mammoth_delivered_envelopes_total gauge"
150
- ]
151
- end
152
-
153
- def metric_line(name, value, destination: nil)
154
- labels = { "mammoth_name" => mammoth_name }
155
- labels["destination"] = destination if destination
156
- %(#{name}{#{labels.map { |label, label_value| %(#{label}="#{escape_label(label_value)}") }.join(",")}} #{Integer(value)})
157
- end
158
-
159
- def escape_label(value)
160
- value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
161
- end
162
161
  end
163
162
  end
@@ -23,6 +23,22 @@ module Mammoth
23
23
  raise NotImplementedError, "#{self.class} must implement #delivered_envelope_store"
24
24
  end
25
25
 
26
+ # Initialize the adapter's durable state.
27
+ #
28
+ # @return [Mammoth::OperationalState::Adapter] self
29
+ def bootstrap!
30
+ raise NotImplementedError, "#{self.class} must implement #bootstrap!"
31
+ end
32
+
33
+ # Report whether the adapter can serve operational state.
34
+ #
35
+ # Implementations must translate backend-specific failures into false.
36
+ #
37
+ # @return [Boolean]
38
+ def ready?
39
+ raise NotImplementedError, "#{self.class} must implement #ready?"
40
+ end
41
+
26
42
  # @return [Hash] JSON-friendly state summary
27
43
  def summary
28
44
  {
@@ -31,6 +31,14 @@ module Mammoth
31
31
  fetch(name).from_config(config)
32
32
  end
33
33
 
34
+ # Build the operational-state adapter selected by configuration.
35
+ #
36
+ # @param config [Mammoth::Configuration] loaded configuration
37
+ # @return [Mammoth::OperationalState::Adapter] adapter instance
38
+ def build_configured(config)
39
+ build(config.dig("operational_state", "adapter") || "sqlite", config)
40
+ end
41
+
34
42
  # @return [Array<String>] registered operational state adapter names
35
43
  def names
36
44
  registry.names
@@ -6,10 +6,10 @@ module Mammoth
6
6
  class SQLiteAdapter < Adapter
7
7
  attr_reader :sqlite_store
8
8
 
9
- # @param sqlite_store [Mammoth::SQLiteStore] bootstrapped SQLite store
9
+ # @param sqlite_store [Mammoth::SQLiteStore] SQLite operational store
10
10
  def initialize(sqlite_store)
11
11
  super()
12
- @sqlite_store = sqlite_store.bootstrap!
12
+ @sqlite_store = sqlite_store
13
13
  end
14
14
 
15
15
  # Build a SQLite state adapter from Mammoth configuration.
@@ -35,6 +35,22 @@ module Mammoth
35
35
  @delivered_envelope_store ||= DeliveredEnvelopeStore.new(sqlite_store)
36
36
  end
37
37
 
38
+ # Initialize the SQLite schema.
39
+ #
40
+ # @return [Mammoth::OperationalState::SQLiteAdapter] self
41
+ def bootstrap!
42
+ sqlite_store.bootstrap!
43
+ self
44
+ end
45
+
46
+ # @return [Boolean] whether SQLite operational state is available
47
+ def ready?
48
+ bootstrap!
49
+ true
50
+ rescue Mammoth::Error, SQLite3::Exception
51
+ false
52
+ end
53
+
38
54
  # @return [Hash] JSON-friendly SQLite state summary
39
55
  def summary
40
56
  super.merge(adapter: "sqlite", path: sqlite_store.path, tables: sqlite_store.tables)