mammoth 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +25 -4
- data/config/mammoth.schema.json +1 -1
- data/lib/mammoth/application.rb +20 -40
- data/lib/mammoth/cli.rb +1 -13
- data/lib/mammoth/commands/bootstrap_command.rb +11 -7
- data/lib/mammoth/commands/dead_letters_command.rb +5 -3
- data/lib/mammoth/commands/deliver_sample_command.rb +1 -1
- data/lib/mammoth/commands/status_command.rb +5 -5
- data/lib/mammoth/concurrent_delivery_runtime.rb +15 -3
- data/lib/mammoth/dead_letter_commands.rb +16 -25
- data/lib/mammoth/dead_letter_store.rb +1 -1
- data/lib/mammoth/delivery_processor.rb +46 -7
- data/lib/mammoth/delivery_worker.rb +10 -8
- data/lib/mammoth/dispatch_metrics.rb +55 -0
- data/lib/mammoth/event_serializer.rb +23 -12
- data/lib/mammoth/fanout_delivery_worker.rb +4 -4
- data/lib/mammoth/metrics_observer.rb +52 -0
- data/lib/mammoth/observability_metrics.rb +69 -0
- data/lib/mammoth/observability_server.rb +6 -6
- data/lib/mammoth/observability_snapshot.rb +59 -60
- data/lib/mammoth/operational_state/adapter.rb +16 -0
- data/lib/mammoth/operational_state/registry.rb +8 -0
- data/lib/mammoth/operational_state/sqlite_adapter.rb +18 -2
- data/lib/mammoth/persisted_payload_deserializer.rb +85 -0
- data/lib/mammoth/replication_consumer.rb +15 -47
- data/lib/mammoth/runtimes/batching_runtime.rb +59 -0
- data/lib/mammoth/runtimes/concurrent_adapter.rb +4 -2
- data/lib/mammoth/runtimes/inline_adapter.rb +19 -5
- data/lib/mammoth/runtimes/registry.rb +3 -2
- data/lib/mammoth/sources/postgres.rb +10 -1
- data/lib/mammoth/status.rb +22 -18
- data/lib/mammoth/transaction_envelope_serializer.rb +13 -15
- data/lib/mammoth/version.rb +1 -1
- data/lib/mammoth/webhook_sink.rb +2 -2
- data/lib/mammoth.rb +8 -1
- metadata +6 -1
|
@@ -17,7 +17,7 @@ module Mammoth
|
|
|
17
17
|
# @param sink [#deliver] destination sink
|
|
18
18
|
# @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
|
|
19
19
|
# @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
|
|
20
|
-
# @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore
|
|
20
|
+
# @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore] downstream delivery ledger
|
|
21
21
|
# @param source_name [String] logical source name
|
|
22
22
|
# @param slot_name [String] replication slot name
|
|
23
23
|
# @param publication_name [String] publication name
|
|
@@ -26,13 +26,13 @@ module Mammoth
|
|
|
26
26
|
# @param sleeper [#call] sleep strategy, injectable for tests
|
|
27
27
|
# @param route_filter [Mammoth::RouteFilter, nil] optional destination route matcher
|
|
28
28
|
# @param enabled [Boolean] whether this destination accepts new deliveries
|
|
29
|
-
def initialize(sink:, checkpoint_store:, dead_letter_store:, source_name:, slot_name:,
|
|
30
|
-
max_attempts:, retry_schedule:,
|
|
29
|
+
def initialize(sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:, source_name:, slot_name:,
|
|
30
|
+
publication_name:, max_attempts:, retry_schedule:, sleeper: Kernel.method(:sleep),
|
|
31
31
|
route_filter: nil, enabled: true)
|
|
32
32
|
@sink = sink
|
|
33
33
|
@checkpoint_store = checkpoint_store
|
|
34
34
|
@dead_letter_store = dead_letter_store
|
|
35
|
-
@delivered_envelope_store = delivered_envelope_store
|
|
35
|
+
@delivered_envelope_store = delivered_envelope_store
|
|
36
36
|
@source_name = source_name
|
|
37
37
|
@slot_name = slot_name
|
|
38
38
|
@publication_name = publication_name
|
|
@@ -49,14 +49,16 @@ module Mammoth
|
|
|
49
49
|
# @param sink [#deliver] destination sink
|
|
50
50
|
# @param checkpoint_store [Mammoth::CheckpointStore] checkpoint persistence
|
|
51
51
|
# @param dead_letter_store [Mammoth::DeadLetterStore] dead letter persistence
|
|
52
|
+
# @param delivered_envelope_store [Mammoth::DeliveredEnvelopeStore] downstream delivery ledger
|
|
52
53
|
# @param sleeper [#call] sleep strategy
|
|
53
54
|
# @return [Mammoth::DeliveryWorker]
|
|
54
|
-
def self.from_config(config, sink:, checkpoint_store:, dead_letter_store:,
|
|
55
|
-
delivery_policy: {})
|
|
55
|
+
def self.from_config(config, sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:,
|
|
56
|
+
sleeper: Kernel.method(:sleep), delivery_policy: {})
|
|
56
57
|
new(
|
|
57
58
|
sink: sink,
|
|
58
59
|
checkpoint_store: checkpoint_store,
|
|
59
60
|
dead_letter_store: dead_letter_store,
|
|
61
|
+
delivered_envelope_store: delivered_envelope_store,
|
|
60
62
|
source_name: config.dig("mammoth", "name"),
|
|
61
63
|
slot_name: config.dig("replication", "slot"),
|
|
62
64
|
publication_name: Array(config.dig("replication", "publications")).join(","),
|
|
@@ -70,7 +72,7 @@ module Mammoth
|
|
|
70
72
|
|
|
71
73
|
# Deliver a transaction envelope with retry, checkpoint, and DLQ handling.
|
|
72
74
|
#
|
|
73
|
-
# @param envelope [
|
|
75
|
+
# @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
|
|
74
76
|
# @return [Hash] delivery summary
|
|
75
77
|
def deliver_transaction(envelope)
|
|
76
78
|
deliver_work(envelope, serializer: TransactionEnvelopeSerializer, delivery_method: :deliver_transaction)
|
|
@@ -78,7 +80,7 @@ module Mammoth
|
|
|
78
80
|
|
|
79
81
|
# Deliver an event with retry, checkpoint, and DLQ handling.
|
|
80
82
|
#
|
|
81
|
-
# @param event [
|
|
83
|
+
# @param event [CDC::Core::ChangeEvent] normalized event
|
|
82
84
|
# @return [Hash] delivery summary
|
|
83
85
|
def deliver(event)
|
|
84
86
|
deliver_work(event, serializer: EventSerializer, delivery_method: :deliver)
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mammoth
|
|
4
|
+
# Thread-safe in-process counters populated by Mammoth's CDC core observer.
|
|
5
|
+
#
|
|
6
|
+
# The registry deliberately stores CDC core metric names and tags. Rendering
|
|
7
|
+
# those counters for a specific backend remains an observability concern.
|
|
8
|
+
class DispatchMetrics
|
|
9
|
+
# Create an empty dispatch metric registry.
|
|
10
|
+
def initialize
|
|
11
|
+
@mutex = Mutex.new
|
|
12
|
+
@counters = Hash.new(0)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Process-wide dispatch metrics used by the default observer and snapshot.
|
|
16
|
+
INSTANCE = new
|
|
17
|
+
|
|
18
|
+
# Increment a canonical CDC core metric.
|
|
19
|
+
#
|
|
20
|
+
# @param name [String] canonical CDC core metric name
|
|
21
|
+
# @param tags [Hash] canonical metric tags
|
|
22
|
+
# @return [Integer] updated counter value
|
|
23
|
+
def increment(name, tags = {})
|
|
24
|
+
key = [name.to_s.freeze, normalized_tags(tags)]
|
|
25
|
+
@mutex.synchronize { @counters[key] += 1 }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Return an immutable point-in-time copy of all counters.
|
|
29
|
+
#
|
|
30
|
+
# @return [Array<Hash>] metric entries with name, tags, and value
|
|
31
|
+
def snapshot
|
|
32
|
+
@mutex.synchronize do
|
|
33
|
+
@counters.map do |(name, tags), value|
|
|
34
|
+
{ name: name, tags: tags.to_h, value: value }
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Clear all counters.
|
|
40
|
+
#
|
|
41
|
+
# Intended for process lifecycle management and isolated tests.
|
|
42
|
+
#
|
|
43
|
+
# @return [self] cleared registry
|
|
44
|
+
def reset!
|
|
45
|
+
@mutex.synchronize { @counters.clear }
|
|
46
|
+
self
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def normalized_tags(tags)
|
|
52
|
+
tags.to_h.transform_keys(&:to_s).sort_by(&:first).freeze
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -15,17 +15,19 @@ module Mammoth
|
|
|
15
15
|
# Default source label used in serialized webhook payloads.
|
|
16
16
|
DEFAULT_SOURCE = "postgresql"
|
|
17
17
|
|
|
18
|
-
# Serialize
|
|
18
|
+
# Serialize a core change event into a webhook-ready Hash.
|
|
19
19
|
#
|
|
20
|
-
# @param event [
|
|
20
|
+
# @param event [CDC::Core::ChangeEvent] normalized CDC event
|
|
21
21
|
# @return [Hash] webhook payload
|
|
22
22
|
def self.call(event)
|
|
23
23
|
new(event).call
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
# @param event [
|
|
26
|
+
# @param event [CDC::Core::ChangeEvent] normalized CDC event
|
|
27
27
|
def initialize(event)
|
|
28
|
-
|
|
28
|
+
raise ArgumentError, "event must be a CDC::Core::ChangeEvent" unless event.is_a?(CDC::Core::ChangeEvent)
|
|
29
|
+
|
|
30
|
+
@event = event.to_h
|
|
29
31
|
end
|
|
30
32
|
|
|
31
33
|
# Return the webhook payload.
|
|
@@ -33,19 +35,20 @@ module Mammoth
|
|
|
33
35
|
# @return [Hash] webhook payload
|
|
34
36
|
def call
|
|
35
37
|
event_hash = stringify_keys(@event)
|
|
38
|
+
metadata = stringify_keys(event_hash["metadata"] || {})
|
|
36
39
|
{
|
|
37
|
-
"event_id" =>
|
|
38
|
-
"source" =>
|
|
40
|
+
"event_id" => event_id(metadata),
|
|
41
|
+
"source" => source(metadata),
|
|
39
42
|
"operation" => normalize_operation(event_hash.fetch("operation")),
|
|
40
|
-
"namespace" => event_hash["
|
|
41
|
-
"entity" => event_hash["
|
|
42
|
-
"identity" => event_hash["
|
|
43
|
-
"source_position" => event_hash["
|
|
43
|
+
"namespace" => event_hash["schema"],
|
|
44
|
+
"entity" => event_hash["table"],
|
|
45
|
+
"identity" => event_hash["primary_key"],
|
|
46
|
+
"source_position" => event_hash["commit_lsn"],
|
|
44
47
|
"transaction_id" => event_hash["transaction_id"],
|
|
45
48
|
"occurred_at" => occurred_at(event_hash),
|
|
46
49
|
"data" => event_data(event_hash),
|
|
47
|
-
"changes" =>
|
|
48
|
-
"metadata" =>
|
|
50
|
+
"changes" => metadata["changes"] || [],
|
|
51
|
+
"metadata" => metadata
|
|
49
52
|
}
|
|
50
53
|
end
|
|
51
54
|
|
|
@@ -66,6 +69,14 @@ module Mammoth
|
|
|
66
69
|
operation.to_s
|
|
67
70
|
end
|
|
68
71
|
|
|
72
|
+
def event_id(metadata)
|
|
73
|
+
metadata["event_id"] || SecureRandom.uuid
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def source(metadata)
|
|
77
|
+
metadata["source"] || DEFAULT_SOURCE
|
|
78
|
+
end
|
|
79
|
+
|
|
69
80
|
def occurred_at(event_hash)
|
|
70
81
|
value = event_hash["occurred_at"]
|
|
71
82
|
return value.iso8601 if value.respond_to?(:iso8601)
|
|
@@ -14,7 +14,7 @@ module Mammoth
|
|
|
14
14
|
|
|
15
15
|
# Deliver an event to every configured destination.
|
|
16
16
|
#
|
|
17
|
-
# @param event [
|
|
17
|
+
# @param event [CDC::Core::ChangeEvent] normalized event
|
|
18
18
|
# @return [Hash] aggregate fanout summary
|
|
19
19
|
def deliver(event)
|
|
20
20
|
fanout(:deliver, event)
|
|
@@ -22,7 +22,7 @@ module Mammoth
|
|
|
22
22
|
|
|
23
23
|
# Deliver a transaction envelope to every configured destination.
|
|
24
24
|
#
|
|
25
|
-
# @param envelope [
|
|
25
|
+
# @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
|
|
26
26
|
# @return [Hash] aggregate fanout summary
|
|
27
27
|
def deliver_transaction(envelope)
|
|
28
28
|
fanout(:deliver_transaction, envelope)
|
|
@@ -31,7 +31,7 @@ module Mammoth
|
|
|
31
31
|
# Deliver an event to one configured destination.
|
|
32
32
|
#
|
|
33
33
|
# @param destination_name [String] destination name
|
|
34
|
-
# @param event [
|
|
34
|
+
# @param event [CDC::Core::ChangeEvent] normalized event
|
|
35
35
|
# @return [Hash] destination delivery summary
|
|
36
36
|
def deliver_to(destination_name, event)
|
|
37
37
|
worker_for(destination_name).deliver(event)
|
|
@@ -40,7 +40,7 @@ module Mammoth
|
|
|
40
40
|
# Deliver a transaction envelope to one configured destination.
|
|
41
41
|
#
|
|
42
42
|
# @param destination_name [String] destination name
|
|
43
|
-
# @param envelope [
|
|
43
|
+
# @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
|
|
44
44
|
# @return [Hash] destination delivery summary
|
|
45
45
|
def deliver_transaction_to(destination_name, envelope)
|
|
46
46
|
worker_for(destination_name).deliver_transaction(envelope)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mammoth
|
|
4
|
+
# CDC core observer that records Mammoth dispatch counters.
|
|
5
|
+
class MetricsObserver < CDC::Core::Observer
|
|
6
|
+
attr_reader :metrics
|
|
7
|
+
|
|
8
|
+
# @param metrics [Mammoth::DispatchMetrics] dispatch counter registry
|
|
9
|
+
def initialize(metrics: DispatchMetrics::INSTANCE)
|
|
10
|
+
super()
|
|
11
|
+
@metrics = metrics
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Record a dispatch attempt.
|
|
15
|
+
#
|
|
16
|
+
# @param event [Object] CDC work item
|
|
17
|
+
# @return [Integer] updated counter
|
|
18
|
+
def dispatch_started(event)
|
|
19
|
+
record(self.class.started_metric_name, event)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Record a successful dispatch.
|
|
23
|
+
#
|
|
24
|
+
# @param result [CDC::Core::ProcessorResult] processor result
|
|
25
|
+
# @return [Integer] updated counter
|
|
26
|
+
def dispatch_succeeded(result)
|
|
27
|
+
record(self.class.succeeded_metric_name, result)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Record a failed dispatch.
|
|
31
|
+
#
|
|
32
|
+
# @param result [CDC::Core::ProcessorResult] processor result
|
|
33
|
+
# @return [Integer] updated counter
|
|
34
|
+
def dispatch_failed(result)
|
|
35
|
+
record(self.class.failed_metric_name, result)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Record a skipped dispatch.
|
|
39
|
+
#
|
|
40
|
+
# @param result [CDC::Core::ProcessorResult] processor result
|
|
41
|
+
# @return [Integer] updated counter
|
|
42
|
+
def dispatch_skipped(result)
|
|
43
|
+
record(self.class.skipped_metric_name, result)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def record(name, payload)
|
|
49
|
+
metrics.increment(name, CDC::Core::Observer.metric_tags(payload))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mammoth
|
|
4
|
+
# Prometheus formatting helpers for operational and dispatch metrics.
|
|
5
|
+
module ObservabilityMetrics
|
|
6
|
+
# Maps canonical CDC core metric names to Mammoth's Prometheus counters.
|
|
7
|
+
DISPATCH_METRIC_NAMES = {
|
|
8
|
+
CDC::Core::Observer.started_metric_name => "mammoth_dispatch_started_total",
|
|
9
|
+
CDC::Core::Observer.succeeded_metric_name => "mammoth_dispatch_succeeded_total",
|
|
10
|
+
CDC::Core::Observer.failed_metric_name => "mammoth_dispatch_failed_total",
|
|
11
|
+
CDC::Core::Observer.skipped_metric_name => "mammoth_dispatch_skipped_total"
|
|
12
|
+
}.freeze
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
def metric_headers
|
|
17
|
+
operational_metric_headers + dispatch_metric_headers
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def operational_metric_headers
|
|
21
|
+
[
|
|
22
|
+
"# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
|
|
23
|
+
"# TYPE mammoth_up gauge",
|
|
24
|
+
"# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
|
|
25
|
+
"# TYPE mammoth_checkpoints_total gauge",
|
|
26
|
+
"# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
|
|
27
|
+
"# TYPE mammoth_dead_letters_total gauge",
|
|
28
|
+
"# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
|
|
29
|
+
"# TYPE mammoth_dead_letters_pending_total gauge",
|
|
30
|
+
"# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
|
|
31
|
+
"# TYPE mammoth_dead_letters_resolved_total gauge",
|
|
32
|
+
"# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
|
|
33
|
+
"# TYPE mammoth_dead_letters_ignored_total gauge",
|
|
34
|
+
"# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
|
|
35
|
+
"# TYPE mammoth_delivered_envelopes_total gauge"
|
|
36
|
+
]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def dispatch_metric_headers
|
|
40
|
+
[
|
|
41
|
+
"# HELP mammoth_dispatch_started_total Number of CDC work items submitted for delivery.",
|
|
42
|
+
"# TYPE mammoth_dispatch_started_total counter",
|
|
43
|
+
"# HELP mammoth_dispatch_succeeded_total Number of successful CDC delivery results.",
|
|
44
|
+
"# TYPE mammoth_dispatch_succeeded_total counter",
|
|
45
|
+
"# HELP mammoth_dispatch_failed_total Number of failed CDC delivery results.",
|
|
46
|
+
"# TYPE mammoth_dispatch_failed_total counter",
|
|
47
|
+
"# HELP mammoth_dispatch_skipped_total Number of skipped CDC delivery results.",
|
|
48
|
+
"# TYPE mammoth_dispatch_skipped_total counter"
|
|
49
|
+
]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def dispatch_metric_lines
|
|
53
|
+
dispatch_metrics.snapshot.sort_by { |entry| [entry.fetch(:name), entry.fetch(:tags).to_a] }.filter_map do |entry|
|
|
54
|
+
name = DISPATCH_METRIC_NAMES[entry.fetch(:name)]
|
|
55
|
+
metric_line(name, entry.fetch(:value), labels: entry.fetch(:tags)) if name
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def metric_line(name, value, destination: nil, labels: {})
|
|
60
|
+
labels = { "mammoth_name" => mammoth_name }.merge(labels.transform_keys(&:to_s))
|
|
61
|
+
labels["destination"] = destination if destination
|
|
62
|
+
%(#{name}{#{labels.map { |label, label_value| %(#{label}="#{escape_label(label_value)}") }.join(",")}} #{Integer(value)})
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def escape_label(value)
|
|
66
|
+
value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -8,25 +8,25 @@ module Mammoth
|
|
|
8
8
|
#
|
|
9
9
|
# The server is intentionally independent from the replication loop. Operators
|
|
10
10
|
# may run it as a sidecar-like process or in a separate process that points at
|
|
11
|
-
# the same
|
|
11
|
+
# the same configured operational-state backend.
|
|
12
12
|
class ObservabilityServer
|
|
13
13
|
# Default bind host for the observability server.
|
|
14
14
|
DEFAULT_HOST = "0.0.0.0"
|
|
15
15
|
# Default TCP port for the observability server.
|
|
16
16
|
DEFAULT_PORT = 9393
|
|
17
17
|
|
|
18
|
-
attr_reader :config, :host, :port, :
|
|
18
|
+
attr_reader :config, :host, :port, :state_adapter, :logger, :server
|
|
19
19
|
|
|
20
20
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
21
21
|
# @param host [String, nil] bind host override
|
|
22
22
|
# @param port [Integer, nil] bind port override
|
|
23
|
-
# @param
|
|
23
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
24
24
|
# @param logger [WEBrick::Log, nil] optional WEBrick logger
|
|
25
|
-
def initialize(config, host: nil, port: nil,
|
|
25
|
+
def initialize(config, host: nil, port: nil, state_adapter: nil, logger: nil)
|
|
26
26
|
@config = config
|
|
27
27
|
@host = host || config.dig("observability", "host") || DEFAULT_HOST
|
|
28
28
|
@port = port || config.dig("observability", "port") || DEFAULT_PORT
|
|
29
|
-
@
|
|
29
|
+
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
30
30
|
@logger = logger || WEBrick::Log.new($stderr, WEBrick::Log::WARN)
|
|
31
31
|
@server = build_server
|
|
32
32
|
mount_endpoints
|
|
@@ -81,7 +81,7 @@ module Mammoth
|
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
def snapshot
|
|
84
|
-
ObservabilitySnapshot.new(config,
|
|
84
|
+
ObservabilitySnapshot.new(config, state_adapter: state_adapter)
|
|
85
85
|
end
|
|
86
86
|
end
|
|
87
87
|
end
|
|
@@ -9,17 +9,21 @@ module Mammoth
|
|
|
9
9
|
# ObservabilitySnapshot is intentionally read-only. It does not start the
|
|
10
10
|
# relay, mutate checkpoints, replay dead letters, or inspect PostgreSQL. The
|
|
11
11
|
# health and metrics endpoints use this object to expose Mammoth process and
|
|
12
|
-
#
|
|
12
|
+
# operational-state adapter status in a predictable format.
|
|
13
13
|
class ObservabilitySnapshot
|
|
14
|
-
|
|
14
|
+
include ObservabilityMetrics
|
|
15
|
+
|
|
16
|
+
attr_reader :config, :state_adapter, :clock, :dispatch_metrics
|
|
15
17
|
|
|
16
18
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
17
|
-
# @param
|
|
19
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
18
20
|
# @param clock [#call] time source returning a Time-like object
|
|
19
|
-
|
|
21
|
+
# @param dispatch_metrics [Mammoth::DispatchMetrics] dispatch counter registry
|
|
22
|
+
def initialize(config, state_adapter: nil, clock: -> { Time.now.utc }, dispatch_metrics: DispatchMetrics::INSTANCE)
|
|
20
23
|
@config = config
|
|
21
|
-
@
|
|
24
|
+
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
22
25
|
@clock = clock
|
|
26
|
+
@dispatch_metrics = dispatch_metrics
|
|
23
27
|
end
|
|
24
28
|
|
|
25
29
|
# Build a liveness response.
|
|
@@ -39,50 +43,55 @@ module Mammoth
|
|
|
39
43
|
#
|
|
40
44
|
# @return [Hash] readiness payload
|
|
41
45
|
def readiness
|
|
42
|
-
|
|
43
|
-
store.bootstrap!
|
|
46
|
+
return unready_payload unless state_adapter.ready?
|
|
44
47
|
|
|
45
48
|
{
|
|
46
49
|
status: "ready",
|
|
47
50
|
service: "mammoth",
|
|
48
51
|
name: mammoth_name,
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
}
|
|
53
|
-
rescue Mammoth::Error, SQLite3::Exception => e
|
|
54
|
-
{
|
|
55
|
-
status: "unready",
|
|
56
|
-
service: "mammoth",
|
|
57
|
-
name: mammoth_name,
|
|
58
|
-
sqlite: "error",
|
|
59
|
-
error_class: e.class.name,
|
|
60
|
-
error_message: e.message,
|
|
52
|
+
operational_state: "ok",
|
|
53
|
+
adapter: state_summary.fetch(:adapter),
|
|
54
|
+
summary: state_summary,
|
|
61
55
|
checked_at: checked_at
|
|
62
56
|
}
|
|
57
|
+
rescue Mammoth::Error => e
|
|
58
|
+
adapter_error_payload(e)
|
|
63
59
|
end
|
|
64
60
|
|
|
65
61
|
# Build a Prometheus text exposition document.
|
|
66
62
|
#
|
|
67
63
|
# @return [String] Prometheus metrics text
|
|
68
64
|
def prometheus
|
|
69
|
-
|
|
70
|
-
checkpoint_store = CheckpointStore.new(store)
|
|
71
|
-
dead_letter_store = DeadLetterStore.new(store)
|
|
72
|
-
delivered_store = DeliveredEnvelopeStore.new(store)
|
|
65
|
+
return down_metrics unless state_adapter.ready?
|
|
73
66
|
|
|
74
67
|
lines = metric_headers + aggregate_metric_lines(
|
|
75
|
-
checkpoint_store: checkpoint_store,
|
|
76
|
-
dead_letter_store: dead_letter_store,
|
|
77
|
-
delivered_store:
|
|
78
|
-
) + destination_metric_lines(
|
|
68
|
+
checkpoint_store: state_adapter.checkpoint_store,
|
|
69
|
+
dead_letter_store: state_adapter.dead_letter_store,
|
|
70
|
+
delivered_store: state_adapter.delivered_envelope_store
|
|
71
|
+
) + destination_metric_lines(
|
|
72
|
+
dead_letter_store: state_adapter.dead_letter_store,
|
|
73
|
+
delivered_store: state_adapter.delivered_envelope_store
|
|
74
|
+
) + dispatch_metric_lines
|
|
79
75
|
"#{lines.join("\n")}\n"
|
|
80
|
-
rescue Mammoth::Error
|
|
81
|
-
|
|
76
|
+
rescue Mammoth::Error
|
|
77
|
+
down_metrics
|
|
82
78
|
end
|
|
83
79
|
|
|
84
80
|
private
|
|
85
81
|
|
|
82
|
+
def adapter_error_payload(error)
|
|
83
|
+
{
|
|
84
|
+
status: "unready",
|
|
85
|
+
service: "mammoth",
|
|
86
|
+
name: mammoth_name,
|
|
87
|
+
operational_state: "error",
|
|
88
|
+
adapter: configured_adapter_name,
|
|
89
|
+
error_class: error.class.name,
|
|
90
|
+
error_message: error.message,
|
|
91
|
+
checked_at: checked_at
|
|
92
|
+
}
|
|
93
|
+
end
|
|
94
|
+
|
|
86
95
|
def aggregate_metric_lines(checkpoint_store:, dead_letter_store:, delivered_store:)
|
|
87
96
|
[
|
|
88
97
|
metric_line("mammoth_up", 1),
|
|
@@ -112,8 +121,27 @@ module Mammoth
|
|
|
112
121
|
end
|
|
113
122
|
end
|
|
114
123
|
|
|
115
|
-
def
|
|
116
|
-
|
|
124
|
+
def unready_payload
|
|
125
|
+
{
|
|
126
|
+
status: "unready",
|
|
127
|
+
service: "mammoth",
|
|
128
|
+
name: mammoth_name,
|
|
129
|
+
operational_state: "error",
|
|
130
|
+
adapter: configured_adapter_name,
|
|
131
|
+
checked_at: checked_at
|
|
132
|
+
}
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def down_metrics
|
|
136
|
+
"#{(metric_headers + [metric_line("mammoth_up", 0)] + dispatch_metric_lines).join("\n")}\n"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def state_summary
|
|
140
|
+
@state_summary ||= state_adapter.summary
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def configured_adapter_name
|
|
144
|
+
config.dig("operational_state", "adapter") || "sqlite"
|
|
117
145
|
end
|
|
118
146
|
|
|
119
147
|
def mammoth_name
|
|
@@ -130,34 +158,5 @@ module Mammoth
|
|
|
130
158
|
def checked_at
|
|
131
159
|
clock.call.utc.iso8601
|
|
132
160
|
end
|
|
133
|
-
|
|
134
|
-
def metric_headers
|
|
135
|
-
[
|
|
136
|
-
"# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
|
|
137
|
-
"# TYPE mammoth_up gauge",
|
|
138
|
-
"# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
|
|
139
|
-
"# TYPE mammoth_checkpoints_total gauge",
|
|
140
|
-
"# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
|
|
141
|
-
"# TYPE mammoth_dead_letters_total gauge",
|
|
142
|
-
"# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
|
|
143
|
-
"# TYPE mammoth_dead_letters_pending_total gauge",
|
|
144
|
-
"# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
|
|
145
|
-
"# TYPE mammoth_dead_letters_resolved_total gauge",
|
|
146
|
-
"# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
|
|
147
|
-
"# TYPE mammoth_dead_letters_ignored_total gauge",
|
|
148
|
-
"# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
|
|
149
|
-
"# TYPE mammoth_delivered_envelopes_total gauge"
|
|
150
|
-
]
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
def metric_line(name, value, destination: nil)
|
|
154
|
-
labels = { "mammoth_name" => mammoth_name }
|
|
155
|
-
labels["destination"] = destination if destination
|
|
156
|
-
%(#{name}{#{labels.map { |label, label_value| %(#{label}="#{escape_label(label_value)}") }.join(",")}} #{Integer(value)})
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
def escape_label(value)
|
|
160
|
-
value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
|
|
161
|
-
end
|
|
162
161
|
end
|
|
163
162
|
end
|
|
@@ -23,6 +23,22 @@ module Mammoth
|
|
|
23
23
|
raise NotImplementedError, "#{self.class} must implement #delivered_envelope_store"
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
# Initialize the adapter's durable state.
|
|
27
|
+
#
|
|
28
|
+
# @return [Mammoth::OperationalState::Adapter] self
|
|
29
|
+
def bootstrap!
|
|
30
|
+
raise NotImplementedError, "#{self.class} must implement #bootstrap!"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Report whether the adapter can serve operational state.
|
|
34
|
+
#
|
|
35
|
+
# Implementations must translate backend-specific failures into false.
|
|
36
|
+
#
|
|
37
|
+
# @return [Boolean]
|
|
38
|
+
def ready?
|
|
39
|
+
raise NotImplementedError, "#{self.class} must implement #ready?"
|
|
40
|
+
end
|
|
41
|
+
|
|
26
42
|
# @return [Hash] JSON-friendly state summary
|
|
27
43
|
def summary
|
|
28
44
|
{
|
|
@@ -31,6 +31,14 @@ module Mammoth
|
|
|
31
31
|
fetch(name).from_config(config)
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
# Build the operational-state adapter selected by configuration.
|
|
35
|
+
#
|
|
36
|
+
# @param config [Mammoth::Configuration] loaded configuration
|
|
37
|
+
# @return [Mammoth::OperationalState::Adapter] adapter instance
|
|
38
|
+
def build_configured(config)
|
|
39
|
+
build(config.dig("operational_state", "adapter") || "sqlite", config)
|
|
40
|
+
end
|
|
41
|
+
|
|
34
42
|
# @return [Array<String>] registered operational state adapter names
|
|
35
43
|
def names
|
|
36
44
|
registry.names
|
|
@@ -6,10 +6,10 @@ module Mammoth
|
|
|
6
6
|
class SQLiteAdapter < Adapter
|
|
7
7
|
attr_reader :sqlite_store
|
|
8
8
|
|
|
9
|
-
# @param sqlite_store [Mammoth::SQLiteStore]
|
|
9
|
+
# @param sqlite_store [Mammoth::SQLiteStore] SQLite operational store
|
|
10
10
|
def initialize(sqlite_store)
|
|
11
11
|
super()
|
|
12
|
-
@sqlite_store = sqlite_store
|
|
12
|
+
@sqlite_store = sqlite_store
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
# Build a SQLite state adapter from Mammoth configuration.
|
|
@@ -35,6 +35,22 @@ module Mammoth
|
|
|
35
35
|
@delivered_envelope_store ||= DeliveredEnvelopeStore.new(sqlite_store)
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
+
# Initialize the SQLite schema.
|
|
39
|
+
#
|
|
40
|
+
# @return [Mammoth::OperationalState::SQLiteAdapter] self
|
|
41
|
+
def bootstrap!
|
|
42
|
+
sqlite_store.bootstrap!
|
|
43
|
+
self
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @return [Boolean] whether SQLite operational state is available
|
|
47
|
+
def ready?
|
|
48
|
+
bootstrap!
|
|
49
|
+
true
|
|
50
|
+
rescue Mammoth::Error, SQLite3::Exception
|
|
51
|
+
false
|
|
52
|
+
end
|
|
53
|
+
|
|
38
54
|
# @return [Hash] JSON-friendly SQLite state summary
|
|
39
55
|
def summary
|
|
40
56
|
super.merge(adapter: "sqlite", path: sqlite_store.path, tables: sqlite_store.tables)
|