mammoth 0.8.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +104 -0
- data/README.md +93 -10
- data/config/mammoth.example.yml +19 -9
- data/config/mammoth.schema.json +3 -3
- data/lib/mammoth/application.rb +54 -43
- data/lib/mammoth/cli.rb +1 -13
- data/lib/mammoth/commands/bootstrap_command.rb +11 -7
- data/lib/mammoth/commands/dead_letters_command.rb +5 -3
- data/lib/mammoth/commands/deliver_sample_command.rb +1 -1
- data/lib/mammoth/commands/status_command.rb +5 -5
- data/lib/mammoth/concurrent_delivery_runtime.rb +15 -3
- data/lib/mammoth/dead_letter_commands.rb +16 -25
- data/lib/mammoth/dead_letter_store.rb +1 -1
- data/lib/mammoth/delivery_processor.rb +51 -9
- data/lib/mammoth/delivery_progress_coordinator.rb +153 -0
- data/lib/mammoth/delivery_worker.rb +19 -33
- data/lib/mammoth/dispatch_metrics.rb +55 -0
- data/lib/mammoth/event_serializer.rb +23 -12
- data/lib/mammoth/fanout_delivery_worker.rb +4 -4
- data/lib/mammoth/metrics_observer.rb +52 -0
- data/lib/mammoth/observability_metrics.rb +71 -0
- data/lib/mammoth/observability_server.rb +8 -6
- data/lib/mammoth/observability_snapshot.rb +118 -66
- data/lib/mammoth/operational_state/adapter.rb +16 -0
- data/lib/mammoth/operational_state/registry.rb +8 -0
- data/lib/mammoth/operational_state/sqlite_adapter.rb +18 -2
- data/lib/mammoth/persisted_payload_deserializer.rb +85 -0
- data/lib/mammoth/postgres_observability_metrics.rb +80 -0
- data/lib/mammoth/replication_consumer.rb +41 -52
- data/lib/mammoth/runtimes/batching_runtime.rb +59 -0
- data/lib/mammoth/runtimes/concurrent_adapter.rb +4 -2
- data/lib/mammoth/runtimes/inline_adapter.rb +19 -5
- data/lib/mammoth/runtimes/registry.rb +3 -2
- data/lib/mammoth/sources/postgres.rb +261 -4
- data/lib/mammoth/sources/postgres_publication_inspector.rb +214 -0
- data/lib/mammoth/sources/postgres_slot_health.rb +90 -0
- data/lib/mammoth/status.rb +22 -18
- data/lib/mammoth/transaction_envelope_serializer.rb +13 -15
- data/lib/mammoth/version.rb +1 -1
- data/lib/mammoth/webhook_sink.rb +2 -2
- data/lib/mammoth.rb +12 -1
- metadata +38 -9
|
@@ -14,7 +14,7 @@ module Mammoth
|
|
|
14
14
|
|
|
15
15
|
# Deliver an event to every configured destination.
|
|
16
16
|
#
|
|
17
|
-
# @param event [
|
|
17
|
+
# @param event [CDC::Core::ChangeEvent] normalized event
|
|
18
18
|
# @return [Hash] aggregate fanout summary
|
|
19
19
|
def deliver(event)
|
|
20
20
|
fanout(:deliver, event)
|
|
@@ -22,7 +22,7 @@ module Mammoth
|
|
|
22
22
|
|
|
23
23
|
# Deliver a transaction envelope to every configured destination.
|
|
24
24
|
#
|
|
25
|
-
# @param envelope [
|
|
25
|
+
# @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
|
|
26
26
|
# @return [Hash] aggregate fanout summary
|
|
27
27
|
def deliver_transaction(envelope)
|
|
28
28
|
fanout(:deliver_transaction, envelope)
|
|
@@ -31,7 +31,7 @@ module Mammoth
|
|
|
31
31
|
# Deliver an event to one configured destination.
|
|
32
32
|
#
|
|
33
33
|
# @param destination_name [String] destination name
|
|
34
|
-
# @param event [
|
|
34
|
+
# @param event [CDC::Core::ChangeEvent] normalized event
|
|
35
35
|
# @return [Hash] destination delivery summary
|
|
36
36
|
def deliver_to(destination_name, event)
|
|
37
37
|
worker_for(destination_name).deliver(event)
|
|
@@ -40,7 +40,7 @@ module Mammoth
|
|
|
40
40
|
# Deliver a transaction envelope to one configured destination.
|
|
41
41
|
#
|
|
42
42
|
# @param destination_name [String] destination name
|
|
43
|
-
# @param envelope [
|
|
43
|
+
# @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
|
|
44
44
|
# @return [Hash] destination delivery summary
|
|
45
45
|
def deliver_transaction_to(destination_name, envelope)
|
|
46
46
|
worker_for(destination_name).deliver_transaction(envelope)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mammoth
|
|
4
|
+
# CDC core observer that records Mammoth dispatch counters.
|
|
5
|
+
class MetricsObserver < CDC::Core::Observer
|
|
6
|
+
attr_reader :metrics
|
|
7
|
+
|
|
8
|
+
# @param metrics [Mammoth::DispatchMetrics] dispatch counter registry
|
|
9
|
+
def initialize(metrics: DispatchMetrics::INSTANCE)
|
|
10
|
+
super()
|
|
11
|
+
@metrics = metrics
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Record a dispatch attempt.
|
|
15
|
+
#
|
|
16
|
+
# @param event [Object] CDC work item
|
|
17
|
+
# @return [Integer] updated counter
|
|
18
|
+
def dispatch_started(event)
|
|
19
|
+
record(self.class.started_metric_name, event)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Record a successful dispatch.
|
|
23
|
+
#
|
|
24
|
+
# @param result [CDC::Core::ProcessorResult] processor result
|
|
25
|
+
# @return [Integer] updated counter
|
|
26
|
+
def dispatch_succeeded(result)
|
|
27
|
+
record(self.class.succeeded_metric_name, result)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Record a failed dispatch.
|
|
31
|
+
#
|
|
32
|
+
# @param result [CDC::Core::ProcessorResult] processor result
|
|
33
|
+
# @return [Integer] updated counter
|
|
34
|
+
def dispatch_failed(result)
|
|
35
|
+
record(self.class.failed_metric_name, result)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Record a skipped dispatch.
|
|
39
|
+
#
|
|
40
|
+
# @param result [CDC::Core::ProcessorResult] processor result
|
|
41
|
+
# @return [Integer] updated counter
|
|
42
|
+
def dispatch_skipped(result)
|
|
43
|
+
record(self.class.skipped_metric_name, result)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def record(name, payload)
|
|
49
|
+
metrics.increment(name, CDC::Core::Observer.metric_tags(payload))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mammoth
|
|
4
|
+
# Prometheus formatting helpers for operational and dispatch metrics.
|
|
5
|
+
module ObservabilityMetrics
|
|
6
|
+
include PostgresObservabilityMetrics
|
|
7
|
+
|
|
8
|
+
# Maps canonical CDC core metric names to Mammoth's Prometheus counters.
|
|
9
|
+
DISPATCH_METRIC_NAMES = {
|
|
10
|
+
CDC::Core::Observer.started_metric_name => "mammoth_dispatch_started_total",
|
|
11
|
+
CDC::Core::Observer.succeeded_metric_name => "mammoth_dispatch_succeeded_total",
|
|
12
|
+
CDC::Core::Observer.failed_metric_name => "mammoth_dispatch_failed_total",
|
|
13
|
+
CDC::Core::Observer.skipped_metric_name => "mammoth_dispatch_skipped_total"
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def metric_headers
|
|
19
|
+
operational_metric_headers + dispatch_metric_headers + postgres_metric_headers
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def operational_metric_headers
|
|
23
|
+
[
|
|
24
|
+
"# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
|
|
25
|
+
"# TYPE mammoth_up gauge",
|
|
26
|
+
"# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
|
|
27
|
+
"# TYPE mammoth_checkpoints_total gauge",
|
|
28
|
+
"# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
|
|
29
|
+
"# TYPE mammoth_dead_letters_total gauge",
|
|
30
|
+
"# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
|
|
31
|
+
"# TYPE mammoth_dead_letters_pending_total gauge",
|
|
32
|
+
"# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
|
|
33
|
+
"# TYPE mammoth_dead_letters_resolved_total gauge",
|
|
34
|
+
"# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
|
|
35
|
+
"# TYPE mammoth_dead_letters_ignored_total gauge",
|
|
36
|
+
"# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
|
|
37
|
+
"# TYPE mammoth_delivered_envelopes_total gauge"
|
|
38
|
+
]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def dispatch_metric_headers
|
|
42
|
+
[
|
|
43
|
+
"# HELP mammoth_dispatch_started_total Number of CDC work items submitted for delivery.",
|
|
44
|
+
"# TYPE mammoth_dispatch_started_total counter",
|
|
45
|
+
"# HELP mammoth_dispatch_succeeded_total Number of successful CDC delivery results.",
|
|
46
|
+
"# TYPE mammoth_dispatch_succeeded_total counter",
|
|
47
|
+
"# HELP mammoth_dispatch_failed_total Number of failed CDC delivery results.",
|
|
48
|
+
"# TYPE mammoth_dispatch_failed_total counter",
|
|
49
|
+
"# HELP mammoth_dispatch_skipped_total Number of skipped CDC delivery results.",
|
|
50
|
+
"# TYPE mammoth_dispatch_skipped_total counter"
|
|
51
|
+
]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def dispatch_metric_lines
|
|
55
|
+
dispatch_metrics.snapshot.sort_by { |entry| [entry.fetch(:name), entry.fetch(:tags).to_a] }.filter_map do |entry|
|
|
56
|
+
name = DISPATCH_METRIC_NAMES[entry.fetch(:name)]
|
|
57
|
+
metric_line(name, entry.fetch(:value), labels: entry.fetch(:tags)) if name
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def metric_line(name, value, destination: nil, labels: {})
|
|
62
|
+
labels = { "mammoth_name" => mammoth_name }.merge(labels.transform_keys(&:to_s))
|
|
63
|
+
labels["destination"] = destination if destination
|
|
64
|
+
%(#{name}{#{labels.map { |label, label_value| %(#{label}="#{escape_label(label_value)}") }.join(",")}} #{Integer(value)})
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def escape_label(value)
|
|
68
|
+
value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -8,25 +8,27 @@ module Mammoth
|
|
|
8
8
|
#
|
|
9
9
|
# The server is intentionally independent from the replication loop. Operators
|
|
10
10
|
# may run it as a sidecar-like process or in a separate process that points at
|
|
11
|
-
# the same
|
|
11
|
+
# the same configured operational-state backend.
|
|
12
12
|
class ObservabilityServer
|
|
13
13
|
# Default bind host for the observability server.
|
|
14
14
|
DEFAULT_HOST = "0.0.0.0"
|
|
15
15
|
# Default TCP port for the observability server.
|
|
16
16
|
DEFAULT_PORT = 9393
|
|
17
17
|
|
|
18
|
-
attr_reader :config, :host, :port, :
|
|
18
|
+
attr_reader :config, :host, :port, :state_adapter, :slot_health_provider, :logger, :server
|
|
19
19
|
|
|
20
20
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
21
21
|
# @param host [String, nil] bind host override
|
|
22
22
|
# @param port [Integer, nil] bind port override
|
|
23
|
-
# @param
|
|
23
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
24
|
+
# @param slot_health_provider [#slot_health, nil] PostgreSQL slot health dependency
|
|
24
25
|
# @param logger [WEBrick::Log, nil] optional WEBrick logger
|
|
25
|
-
def initialize(config, host: nil, port: nil,
|
|
26
|
+
def initialize(config, host: nil, port: nil, state_adapter: nil, slot_health_provider: nil, logger: nil)
|
|
26
27
|
@config = config
|
|
27
28
|
@host = host || config.dig("observability", "host") || DEFAULT_HOST
|
|
28
29
|
@port = port || config.dig("observability", "port") || DEFAULT_PORT
|
|
29
|
-
@
|
|
30
|
+
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
31
|
+
@slot_health_provider = slot_health_provider || Sources::Postgres.new(config)
|
|
30
32
|
@logger = logger || WEBrick::Log.new($stderr, WEBrick::Log::WARN)
|
|
31
33
|
@server = build_server
|
|
32
34
|
mount_endpoints
|
|
@@ -81,7 +83,7 @@ module Mammoth
|
|
|
81
83
|
end
|
|
82
84
|
|
|
83
85
|
def snapshot
|
|
84
|
-
ObservabilitySnapshot.new(config,
|
|
86
|
+
ObservabilitySnapshot.new(config, state_adapter:, slot_health_provider:)
|
|
85
87
|
end
|
|
86
88
|
end
|
|
87
89
|
end
|
|
@@ -3,23 +3,30 @@
|
|
|
3
3
|
require "time"
|
|
4
4
|
|
|
5
5
|
module Mammoth
|
|
6
|
-
# Builds health, readiness, and metrics snapshots from Mammoth's
|
|
7
|
-
#
|
|
6
|
+
# Builds health, readiness, and metrics snapshots from Mammoth's operational
|
|
7
|
+
# state and optional PostgreSQL slot-health provider.
|
|
8
8
|
#
|
|
9
9
|
# ObservabilitySnapshot is intentionally read-only. It does not start the
|
|
10
|
-
# relay, mutate checkpoints, replay dead letters, or
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
class ObservabilitySnapshot
|
|
14
|
-
|
|
10
|
+
# relay, mutate checkpoints, replay dead letters, or change PostgreSQL slot
|
|
11
|
+
# state. An injected provider may perform read-only slot inspection. The
|
|
12
|
+
# endpoints expose process, operational-state, and source status predictably.
|
|
13
|
+
class ObservabilitySnapshot # rubocop:disable Metrics/ClassLength
|
|
14
|
+
include ObservabilityMetrics
|
|
15
|
+
|
|
16
|
+
attr_reader :config, :state_adapter, :clock, :dispatch_metrics, :slot_health_provider
|
|
15
17
|
|
|
16
18
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
17
|
-
# @param
|
|
19
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
18
20
|
# @param clock [#call] time source returning a Time-like object
|
|
19
|
-
|
|
21
|
+
# @param dispatch_metrics [Mammoth::DispatchMetrics] dispatch counter registry
|
|
22
|
+
# @param slot_health_provider [#slot_health, nil] PostgreSQL slot health dependency
|
|
23
|
+
def initialize(config, state_adapter: nil, clock: -> { Time.now.utc }, dispatch_metrics: DispatchMetrics::INSTANCE,
|
|
24
|
+
slot_health_provider: nil)
|
|
20
25
|
@config = config
|
|
21
|
-
@
|
|
26
|
+
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
22
27
|
@clock = clock
|
|
28
|
+
@dispatch_metrics = dispatch_metrics
|
|
29
|
+
@slot_health_provider = slot_health_provider
|
|
23
30
|
end
|
|
24
31
|
|
|
25
32
|
# Build a liveness response.
|
|
@@ -39,50 +46,93 @@ module Mammoth
|
|
|
39
46
|
#
|
|
40
47
|
# @return [Hash] readiness payload
|
|
41
48
|
def readiness
|
|
42
|
-
|
|
43
|
-
store.bootstrap!
|
|
49
|
+
return unready_payload unless state_adapter.ready?
|
|
44
50
|
|
|
45
|
-
{
|
|
51
|
+
payload = {
|
|
46
52
|
status: "ready",
|
|
47
53
|
service: "mammoth",
|
|
48
54
|
name: mammoth_name,
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
}
|
|
53
|
-
rescue Mammoth::Error, SQLite3::Exception => e
|
|
54
|
-
{
|
|
55
|
-
status: "unready",
|
|
56
|
-
service: "mammoth",
|
|
57
|
-
name: mammoth_name,
|
|
58
|
-
sqlite: "error",
|
|
59
|
-
error_class: e.class.name,
|
|
60
|
-
error_message: e.message,
|
|
55
|
+
operational_state: "ok",
|
|
56
|
+
adapter: state_summary.fetch(:adapter),
|
|
57
|
+
summary: state_summary,
|
|
61
58
|
checked_at: checked_at
|
|
62
59
|
}
|
|
60
|
+
return payload unless slot_health_provider
|
|
61
|
+
|
|
62
|
+
health = postgres_slot_health
|
|
63
|
+
return postgres_unready_payload(health) unless health.ready?
|
|
64
|
+
|
|
65
|
+
payload.merge(postgres_slot: health.summary)
|
|
66
|
+
rescue ReplicationError => e
|
|
67
|
+
postgres_error_payload(e)
|
|
68
|
+
rescue Mammoth::Error => e
|
|
69
|
+
adapter_error_payload(e)
|
|
63
70
|
end
|
|
64
71
|
|
|
65
72
|
# Build a Prometheus text exposition document.
|
|
66
73
|
#
|
|
67
74
|
# @return [String] Prometheus metrics text
|
|
68
75
|
def prometheus
|
|
69
|
-
|
|
70
|
-
checkpoint_store = CheckpointStore.new(store)
|
|
71
|
-
dead_letter_store = DeadLetterStore.new(store)
|
|
72
|
-
delivered_store = DeliveredEnvelopeStore.new(store)
|
|
76
|
+
return down_metrics unless state_adapter.ready?
|
|
73
77
|
|
|
74
78
|
lines = metric_headers + aggregate_metric_lines(
|
|
75
|
-
checkpoint_store: checkpoint_store,
|
|
76
|
-
dead_letter_store: dead_letter_store,
|
|
77
|
-
delivered_store:
|
|
78
|
-
) + destination_metric_lines(
|
|
79
|
+
checkpoint_store: state_adapter.checkpoint_store,
|
|
80
|
+
dead_letter_store: state_adapter.dead_letter_store,
|
|
81
|
+
delivered_store: state_adapter.delivered_envelope_store
|
|
82
|
+
) + destination_metric_lines(
|
|
83
|
+
dead_letter_store: state_adapter.dead_letter_store,
|
|
84
|
+
delivered_store: state_adapter.delivered_envelope_store
|
|
85
|
+
) + dispatch_metric_lines + postgres_slot_metric_lines
|
|
79
86
|
"#{lines.join("\n")}\n"
|
|
80
|
-
rescue Mammoth::Error
|
|
81
|
-
|
|
87
|
+
rescue Mammoth::Error
|
|
88
|
+
down_metrics
|
|
82
89
|
end
|
|
83
90
|
|
|
84
91
|
private
|
|
85
92
|
|
|
93
|
+
def adapter_error_payload(error)
|
|
94
|
+
{
|
|
95
|
+
status: "unready",
|
|
96
|
+
service: "mammoth",
|
|
97
|
+
name: mammoth_name,
|
|
98
|
+
operational_state: "error",
|
|
99
|
+
adapter: configured_adapter_name,
|
|
100
|
+
error_class: error.class.name,
|
|
101
|
+
error_message: error.message,
|
|
102
|
+
checked_at: checked_at
|
|
103
|
+
}
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def postgres_unready_payload(health)
|
|
107
|
+
{
|
|
108
|
+
status: "unready",
|
|
109
|
+
service: "mammoth",
|
|
110
|
+
name: mammoth_name,
|
|
111
|
+
operational_state: "ok",
|
|
112
|
+
adapter: state_summary.fetch(:adapter),
|
|
113
|
+
summary: state_summary,
|
|
114
|
+
postgres_slot: health.summary,
|
|
115
|
+
checked_at: checked_at
|
|
116
|
+
}
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def postgres_error_payload(error)
|
|
120
|
+
{
|
|
121
|
+
status: "unready",
|
|
122
|
+
service: "mammoth",
|
|
123
|
+
name: mammoth_name,
|
|
124
|
+
operational_state: "ok",
|
|
125
|
+
adapter: configured_adapter_name,
|
|
126
|
+
postgres_slot: {
|
|
127
|
+
ready: false,
|
|
128
|
+
reason: "inspection failed",
|
|
129
|
+
error_class: error.class.name,
|
|
130
|
+
error_message: error.message
|
|
131
|
+
},
|
|
132
|
+
checked_at: checked_at
|
|
133
|
+
}
|
|
134
|
+
end
|
|
135
|
+
|
|
86
136
|
def aggregate_metric_lines(checkpoint_store:, dead_letter_store:, delivered_store:)
|
|
87
137
|
[
|
|
88
138
|
metric_line("mammoth_up", 1),
|
|
@@ -112,8 +162,39 @@ module Mammoth
|
|
|
112
162
|
end
|
|
113
163
|
end
|
|
114
164
|
|
|
115
|
-
def
|
|
116
|
-
|
|
165
|
+
def unready_payload
|
|
166
|
+
{
|
|
167
|
+
status: "unready",
|
|
168
|
+
service: "mammoth",
|
|
169
|
+
name: mammoth_name,
|
|
170
|
+
operational_state: "error",
|
|
171
|
+
adapter: configured_adapter_name,
|
|
172
|
+
checked_at: checked_at
|
|
173
|
+
}
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def down_metrics
|
|
177
|
+
"#{(metric_headers + [metric_line("mammoth_up", 0)] + dispatch_metric_lines).join("\n")}\n"
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def postgres_slot_metric_lines
|
|
181
|
+
return [] unless slot_health_provider
|
|
182
|
+
|
|
183
|
+
postgres_metric_lines(postgres_slot_health)
|
|
184
|
+
rescue ReplicationError
|
|
185
|
+
postgres_inspection_error_metric_lines
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def postgres_slot_health
|
|
189
|
+
slot_health_provider.slot_health
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def state_summary
|
|
193
|
+
@state_summary ||= state_adapter.summary
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def configured_adapter_name
|
|
197
|
+
config.dig("operational_state", "adapter") || "sqlite"
|
|
117
198
|
end
|
|
118
199
|
|
|
119
200
|
def mammoth_name
|
|
@@ -130,34 +211,5 @@ module Mammoth
|
|
|
130
211
|
def checked_at
|
|
131
212
|
clock.call.utc.iso8601
|
|
132
213
|
end
|
|
133
|
-
|
|
134
|
-
def metric_headers
|
|
135
|
-
[
|
|
136
|
-
"# HELP mammoth_up 1 when Mammoth operational state can be inspected, 0 otherwise.",
|
|
137
|
-
"# TYPE mammoth_up gauge",
|
|
138
|
-
"# HELP mammoth_checkpoints_total Number of checkpoint rows stored by Mammoth.",
|
|
139
|
-
"# TYPE mammoth_checkpoints_total gauge",
|
|
140
|
-
"# HELP mammoth_dead_letters_total Number of dead-letter rows stored by Mammoth.",
|
|
141
|
-
"# TYPE mammoth_dead_letters_total gauge",
|
|
142
|
-
"# HELP mammoth_dead_letters_pending_total Number of pending dead-letter rows.",
|
|
143
|
-
"# TYPE mammoth_dead_letters_pending_total gauge",
|
|
144
|
-
"# HELP mammoth_dead_letters_resolved_total Number of resolved dead-letter rows.",
|
|
145
|
-
"# TYPE mammoth_dead_letters_resolved_total gauge",
|
|
146
|
-
"# HELP mammoth_dead_letters_ignored_total Number of ignored dead-letter rows.",
|
|
147
|
-
"# TYPE mammoth_dead_letters_ignored_total gauge",
|
|
148
|
-
"# HELP mammoth_delivered_envelopes_total Number of delivered-envelope ledger rows.",
|
|
149
|
-
"# TYPE mammoth_delivered_envelopes_total gauge"
|
|
150
|
-
]
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
def metric_line(name, value, destination: nil)
|
|
154
|
-
labels = { "mammoth_name" => mammoth_name }
|
|
155
|
-
labels["destination"] = destination if destination
|
|
156
|
-
%(#{name}{#{labels.map { |label, label_value| %(#{label}="#{escape_label(label_value)}") }.join(",")}} #{Integer(value)})
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
def escape_label(value)
|
|
160
|
-
value.to_s.gsub("\\", "\\\\").gsub('"', '\\"').gsub("\n", "\\n")
|
|
161
|
-
end
|
|
162
214
|
end
|
|
163
215
|
end
|
|
@@ -23,6 +23,22 @@ module Mammoth
|
|
|
23
23
|
raise NotImplementedError, "#{self.class} must implement #delivered_envelope_store"
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
# Initialize the adapter's durable state.
|
|
27
|
+
#
|
|
28
|
+
# @return [Mammoth::OperationalState::Adapter] self
|
|
29
|
+
def bootstrap!
|
|
30
|
+
raise NotImplementedError, "#{self.class} must implement #bootstrap!"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Report whether the adapter can serve operational state.
|
|
34
|
+
#
|
|
35
|
+
# Implementations must translate backend-specific failures into false.
|
|
36
|
+
#
|
|
37
|
+
# @return [Boolean]
|
|
38
|
+
def ready?
|
|
39
|
+
raise NotImplementedError, "#{self.class} must implement #ready?"
|
|
40
|
+
end
|
|
41
|
+
|
|
26
42
|
# @return [Hash] JSON-friendly state summary
|
|
27
43
|
def summary
|
|
28
44
|
{
|
|
@@ -31,6 +31,14 @@ module Mammoth
|
|
|
31
31
|
fetch(name).from_config(config)
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
# Build the operational-state adapter selected by configuration.
|
|
35
|
+
#
|
|
36
|
+
# @param config [Mammoth::Configuration] loaded configuration
|
|
37
|
+
# @return [Mammoth::OperationalState::Adapter] adapter instance
|
|
38
|
+
def build_configured(config)
|
|
39
|
+
build(config.dig("operational_state", "adapter") || "sqlite", config)
|
|
40
|
+
end
|
|
41
|
+
|
|
34
42
|
# @return [Array<String>] registered operational state adapter names
|
|
35
43
|
def names
|
|
36
44
|
registry.names
|
|
@@ -6,10 +6,10 @@ module Mammoth
|
|
|
6
6
|
class SQLiteAdapter < Adapter
|
|
7
7
|
attr_reader :sqlite_store
|
|
8
8
|
|
|
9
|
-
# @param sqlite_store [Mammoth::SQLiteStore]
|
|
9
|
+
# @param sqlite_store [Mammoth::SQLiteStore] SQLite operational store
|
|
10
10
|
def initialize(sqlite_store)
|
|
11
11
|
super()
|
|
12
|
-
@sqlite_store = sqlite_store
|
|
12
|
+
@sqlite_store = sqlite_store
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
# Build a SQLite state adapter from Mammoth configuration.
|
|
@@ -35,6 +35,22 @@ module Mammoth
|
|
|
35
35
|
@delivered_envelope_store ||= DeliveredEnvelopeStore.new(sqlite_store)
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
+
# Initialize the SQLite schema.
|
|
39
|
+
#
|
|
40
|
+
# @return [Mammoth::OperationalState::SQLiteAdapter] self
|
|
41
|
+
def bootstrap!
|
|
42
|
+
sqlite_store.bootstrap!
|
|
43
|
+
self
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @return [Boolean] whether SQLite operational state is available
|
|
47
|
+
def ready?
|
|
48
|
+
bootstrap!
|
|
49
|
+
true
|
|
50
|
+
rescue Mammoth::Error, SQLite3::Exception
|
|
51
|
+
false
|
|
52
|
+
end
|
|
53
|
+
|
|
38
54
|
# @return [Hash] JSON-friendly SQLite state summary
|
|
39
55
|
def summary
|
|
40
56
|
super.merge(adapter: "sqlite", path: sqlite_store.path, tables: sqlite_store.tables)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module Mammoth
|
|
6
|
+
# Reconstructs exact CDC core work items from persisted webhook payloads.
|
|
7
|
+
#
|
|
8
|
+
# Dead letters and sample files contain Mammoth's JSON delivery projection,
|
|
9
|
+
# not live CDC objects. This class is the explicit boundary that converts
|
|
10
|
+
# those persisted representations back into core vocabulary.
|
|
11
|
+
class PersistedPayloadDeserializer
|
|
12
|
+
class << self
|
|
13
|
+
# Deserialize one persisted event payload.
|
|
14
|
+
#
|
|
15
|
+
# @param payload [Hash] Mammoth event payload
|
|
16
|
+
# @return [CDC::Core::ChangeEvent] reconstructed core event
|
|
17
|
+
def event(payload)
|
|
18
|
+
attributes = stringify_keys(payload)
|
|
19
|
+
data = attributes["data"]
|
|
20
|
+
metadata = enriched_metadata(attributes)
|
|
21
|
+
|
|
22
|
+
CDC::Core::ChangeEvent.new(
|
|
23
|
+
operation: attributes.fetch("operation"),
|
|
24
|
+
schema: attributes["namespace"] || attributes.fetch("schema"),
|
|
25
|
+
table: attributes["entity"] || attributes.fetch("table"),
|
|
26
|
+
old_values: attributes["old_values"] || delete_values(attributes, data),
|
|
27
|
+
new_values: attributes["new_values"] || changed_values(attributes, data),
|
|
28
|
+
primary_key: attributes["identity"] || attributes["primary_key"],
|
|
29
|
+
transaction_id: attributes["transaction_id"],
|
|
30
|
+
commit_lsn: attributes["commit_lsn"] || attributes["source_position"],
|
|
31
|
+
sequence_number: attributes["sequence_number"],
|
|
32
|
+
occurred_at: parse_time(attributes["occurred_at"]),
|
|
33
|
+
metadata: metadata
|
|
34
|
+
)
|
|
35
|
+
rescue KeyError, ArgumentError, TypeError => e
|
|
36
|
+
raise ConfigurationError, "invalid persisted CDC event: #{e.message}"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Deserialize one persisted transaction payload.
|
|
40
|
+
#
|
|
41
|
+
# @param payload [Hash] Mammoth transaction webhook payload
|
|
42
|
+
# @return [CDC::Core::TransactionEnvelope] reconstructed core transaction
|
|
43
|
+
def transaction(payload)
|
|
44
|
+
attributes = stringify_keys(payload)
|
|
45
|
+
CDC::Core::TransactionEnvelope.new(
|
|
46
|
+
transaction_id: attributes.fetch("transaction_id"),
|
|
47
|
+
events: attributes.fetch("events").map { |event_payload| event(event_payload) },
|
|
48
|
+
commit_lsn: attributes["commit_lsn"] || attributes["source_position"],
|
|
49
|
+
committed_at: parse_time(attributes["committed_at"]),
|
|
50
|
+
metadata: enriched_metadata(attributes)
|
|
51
|
+
)
|
|
52
|
+
rescue KeyError, ArgumentError, TypeError => e
|
|
53
|
+
raise ConfigurationError, "invalid persisted CDC transaction: #{e.message}"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def stringify_keys(payload)
|
|
59
|
+
payload.to_h.transform_keys(&:to_s)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def enriched_metadata(attributes)
|
|
63
|
+
metadata = stringify_keys(attributes["metadata"] || {})
|
|
64
|
+
%w[event_id source source_position changes type].each do |key|
|
|
65
|
+
metadata[key] = attributes[key] if attributes.key?(key)
|
|
66
|
+
end
|
|
67
|
+
metadata
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def delete_values(attributes, data)
|
|
71
|
+
data if attributes["operation"].to_s == "delete"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def changed_values(attributes, data)
|
|
75
|
+
data unless attributes["operation"].to_s == "delete"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def parse_time(value)
|
|
79
|
+
return value if value.nil? || value.is_a?(Time)
|
|
80
|
+
|
|
81
|
+
Time.iso8601(value.to_s)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mammoth
|
|
4
|
+
# PostgreSQL slot-specific Prometheus formatting helpers.
|
|
5
|
+
module PostgresObservabilityMetrics
|
|
6
|
+
# Metric names and help text for PostgreSQL replication-slot gauges.
|
|
7
|
+
POSTGRES_METRICS = {
|
|
8
|
+
"mammoth_postgres_slot_inspection_up" => "1 when PostgreSQL slot inspection succeeds.",
|
|
9
|
+
"mammoth_postgres_slot_present" => "1 when the configured replication slot exists.",
|
|
10
|
+
"mammoth_postgres_slot_ready" => "1 when the configured slot is healthy and active.",
|
|
11
|
+
"mammoth_postgres_slot_active" => "1 when PostgreSQL reports the slot as active.",
|
|
12
|
+
"mammoth_postgres_slot_retained_wal_bytes" => "WAL retained from the slot restart LSN.",
|
|
13
|
+
"mammoth_postgres_slot_safe_wal_size_bytes" => "WAL bytes that may be written before the slot risks loss.",
|
|
14
|
+
"mammoth_postgres_slot_wal_status" => "Current PostgreSQL WAL retention status for the slot.",
|
|
15
|
+
"mammoth_postgres_slot_invalidated" => "1 when PostgreSQL reports invalidation or conflict.",
|
|
16
|
+
"mammoth_postgres_slot_inactive_since_timestamp_seconds" => "Unix timestamp when the slot became inactive.",
|
|
17
|
+
"mammoth_postgres_slot_restart_lsn_bytes" => "Numeric PostgreSQL restart LSN.",
|
|
18
|
+
"mammoth_postgres_slot_confirmed_flush_lsn_bytes" => "Numeric PostgreSQL confirmed flush LSN."
|
|
19
|
+
}.freeze
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def postgres_metric_headers
|
|
24
|
+
POSTGRES_METRICS.flat_map { |name, help| ["# HELP #{name} #{help}", "# TYPE #{name} gauge"] }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def postgres_metric_lines(health)
|
|
28
|
+
labels = { slot_name: health.slot_name }
|
|
29
|
+
lines = postgres_presence_metric_lines(health, labels:)
|
|
30
|
+
return lines unless health.present
|
|
31
|
+
|
|
32
|
+
lines + postgres_status_metric_lines(health, labels:) + postgres_optional_metric_lines(health, labels:)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def postgres_presence_metric_lines(health, labels:)
|
|
36
|
+
[
|
|
37
|
+
metric_line("mammoth_postgres_slot_inspection_up", 1, labels:),
|
|
38
|
+
metric_line("mammoth_postgres_slot_present", health.present ? 1 : 0, labels:)
|
|
39
|
+
]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def postgres_status_metric_lines(health, labels:)
|
|
43
|
+
[
|
|
44
|
+
metric_line("mammoth_postgres_slot_ready", health.ready? ? 1 : 0, labels:),
|
|
45
|
+
metric_line("mammoth_postgres_slot_active", health.active ? 1 : 0, labels:),
|
|
46
|
+
metric_line("mammoth_postgres_slot_wal_status", 1,
|
|
47
|
+
labels: labels.merge(wal_status: health.wal_status || "unknown")),
|
|
48
|
+
metric_line("mammoth_postgres_slot_invalidated", invalidated?(health) ? 1 : 0, labels:)
|
|
49
|
+
]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def postgres_optional_metric_lines(health, labels:)
|
|
53
|
+
{
|
|
54
|
+
"mammoth_postgres_slot_retained_wal_bytes" => health.retained_wal_bytes,
|
|
55
|
+
"mammoth_postgres_slot_safe_wal_size_bytes" => health.safe_wal_size,
|
|
56
|
+
"mammoth_postgres_slot_inactive_since_timestamp_seconds" => timestamp_seconds(health.inactive_since),
|
|
57
|
+
"mammoth_postgres_slot_restart_lsn_bytes" => health.restart_lsn_bytes,
|
|
58
|
+
"mammoth_postgres_slot_confirmed_flush_lsn_bytes" => health.confirmed_flush_lsn_bytes
|
|
59
|
+
}.filter_map { |name, value| metric_line(name, value, labels:) unless value.nil? }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def postgres_inspection_error_metric_lines
|
|
63
|
+
labels = { slot_name: config.dig("replication", "slot") }
|
|
64
|
+
[metric_line("mammoth_postgres_slot_inspection_up", 0, labels:)]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def timestamp_seconds(value)
|
|
68
|
+
return nil if value.nil?
|
|
69
|
+
return value.to_i if value.respond_to?(:to_i) && !value.is_a?(String)
|
|
70
|
+
|
|
71
|
+
Time.parse(value.to_s).to_i
|
|
72
|
+
rescue ArgumentError
|
|
73
|
+
nil
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def invalidated?(health)
|
|
77
|
+
health.conflicting || (!health.invalidation_reason.nil? && health.invalidation_reason != "")
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|