mammoth 0.9.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +66 -0
- data/README.md +69 -7
- data/config/mammoth.example.yml +19 -9
- data/config/mammoth.schema.json +2 -2
- data/lib/mammoth/application.rb +34 -3
- data/lib/mammoth/delivery_processor.rb +5 -2
- data/lib/mammoth/delivery_progress_coordinator.rb +153 -0
- data/lib/mammoth/delivery_worker.rb +9 -25
- data/lib/mammoth/observability_metrics.rb +3 -1
- data/lib/mammoth/observability_server.rb +5 -3
- data/lib/mammoth/observability_snapshot.rb +63 -10
- data/lib/mammoth/postgres_observability_metrics.rb +80 -0
- data/lib/mammoth/replication_consumer.rb +28 -7
- data/lib/mammoth/sources/postgres.rb +252 -4
- data/lib/mammoth/sources/postgres_publication_inspector.rb +214 -0
- data/lib/mammoth/sources/postgres_slot_health.rb +90 -0
- data/lib/mammoth/version.rb +1 -1
- data/lib/mammoth.rb +5 -1
- metadata +33 -9
|
@@ -15,18 +15,20 @@ module Mammoth
|
|
|
15
15
|
# Default TCP port for the observability server.
|
|
16
16
|
DEFAULT_PORT = 9393
|
|
17
17
|
|
|
18
|
-
attr_reader :config, :host, :port, :state_adapter, :logger, :server
|
|
18
|
+
attr_reader :config, :host, :port, :state_adapter, :slot_health_provider, :logger, :server
|
|
19
19
|
|
|
20
20
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
21
21
|
# @param host [String, nil] bind host override
|
|
22
22
|
# @param port [Integer, nil] bind port override
|
|
23
23
|
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
24
|
+
# @param slot_health_provider [#slot_health, nil] PostgreSQL slot health dependency
|
|
24
25
|
# @param logger [WEBrick::Log, nil] optional WEBrick logger
|
|
25
|
-
def initialize(config, host: nil, port: nil, state_adapter: nil, logger: nil)
|
|
26
|
+
def initialize(config, host: nil, port: nil, state_adapter: nil, slot_health_provider: nil, logger: nil)
|
|
26
27
|
@config = config
|
|
27
28
|
@host = host || config.dig("observability", "host") || DEFAULT_HOST
|
|
28
29
|
@port = port || config.dig("observability", "port") || DEFAULT_PORT
|
|
29
30
|
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
31
|
+
@slot_health_provider = slot_health_provider || Sources::Postgres.new(config)
|
|
30
32
|
@logger = logger || WEBrick::Log.new($stderr, WEBrick::Log::WARN)
|
|
31
33
|
@server = build_server
|
|
32
34
|
mount_endpoints
|
|
@@ -81,7 +83,7 @@ module Mammoth
|
|
|
81
83
|
end
|
|
82
84
|
|
|
83
85
|
def snapshot
|
|
84
|
-
ObservabilitySnapshot.new(config, state_adapter:
|
|
86
|
+
ObservabilitySnapshot.new(config, state_adapter:, slot_health_provider:)
|
|
85
87
|
end
|
|
86
88
|
end
|
|
87
89
|
end
|
|
@@ -3,27 +3,30 @@
|
|
|
3
3
|
require "time"
|
|
4
4
|
|
|
5
5
|
module Mammoth
|
|
6
|
-
# Builds health, readiness, and metrics snapshots from Mammoth's
|
|
7
|
-
#
|
|
6
|
+
# Builds health, readiness, and metrics snapshots from Mammoth's operational
|
|
7
|
+
# state and optional PostgreSQL slot-health provider.
|
|
8
8
|
#
|
|
9
9
|
# ObservabilitySnapshot is intentionally read-only. It does not start the
|
|
10
|
-
# relay, mutate checkpoints, replay dead letters, or
|
|
11
|
-
#
|
|
12
|
-
# operational-state
|
|
13
|
-
class ObservabilitySnapshot
|
|
10
|
+
# relay, mutate checkpoints, replay dead letters, or change PostgreSQL slot
|
|
11
|
+
# state. An injected provider may perform read-only slot inspection. The
|
|
12
|
+
# endpoints expose process, operational-state, and source status predictably.
|
|
13
|
+
class ObservabilitySnapshot # rubocop:disable Metrics/ClassLength
|
|
14
14
|
include ObservabilityMetrics
|
|
15
15
|
|
|
16
|
-
attr_reader :config, :state_adapter, :clock, :dispatch_metrics
|
|
16
|
+
attr_reader :config, :state_adapter, :clock, :dispatch_metrics, :slot_health_provider
|
|
17
17
|
|
|
18
18
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
19
19
|
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
20
20
|
# @param clock [#call] time source returning a Time-like object
|
|
21
21
|
# @param dispatch_metrics [Mammoth::DispatchMetrics] dispatch counter registry
|
|
22
|
-
|
|
22
|
+
# @param slot_health_provider [#slot_health, nil] PostgreSQL slot health dependency
|
|
23
|
+
def initialize(config, state_adapter: nil, clock: -> { Time.now.utc }, dispatch_metrics: DispatchMetrics::INSTANCE,
|
|
24
|
+
slot_health_provider: nil)
|
|
23
25
|
@config = config
|
|
24
26
|
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
25
27
|
@clock = clock
|
|
26
28
|
@dispatch_metrics = dispatch_metrics
|
|
29
|
+
@slot_health_provider = slot_health_provider
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
# Build a liveness response.
|
|
@@ -45,7 +48,7 @@ module Mammoth
|
|
|
45
48
|
def readiness
|
|
46
49
|
return unready_payload unless state_adapter.ready?
|
|
47
50
|
|
|
48
|
-
{
|
|
51
|
+
payload = {
|
|
49
52
|
status: "ready",
|
|
50
53
|
service: "mammoth",
|
|
51
54
|
name: mammoth_name,
|
|
@@ -54,6 +57,14 @@ module Mammoth
|
|
|
54
57
|
summary: state_summary,
|
|
55
58
|
checked_at: checked_at
|
|
56
59
|
}
|
|
60
|
+
return payload unless slot_health_provider
|
|
61
|
+
|
|
62
|
+
health = postgres_slot_health
|
|
63
|
+
return postgres_unready_payload(health) unless health.ready?
|
|
64
|
+
|
|
65
|
+
payload.merge(postgres_slot: health.summary)
|
|
66
|
+
rescue ReplicationError => e
|
|
67
|
+
postgres_error_payload(e)
|
|
57
68
|
rescue Mammoth::Error => e
|
|
58
69
|
adapter_error_payload(e)
|
|
59
70
|
end
|
|
@@ -71,7 +82,7 @@ module Mammoth
|
|
|
71
82
|
) + destination_metric_lines(
|
|
72
83
|
dead_letter_store: state_adapter.dead_letter_store,
|
|
73
84
|
delivered_store: state_adapter.delivered_envelope_store
|
|
74
|
-
) + dispatch_metric_lines
|
|
85
|
+
) + dispatch_metric_lines + postgres_slot_metric_lines
|
|
75
86
|
"#{lines.join("\n")}\n"
|
|
76
87
|
rescue Mammoth::Error
|
|
77
88
|
down_metrics
|
|
@@ -92,6 +103,36 @@ module Mammoth
|
|
|
92
103
|
}
|
|
93
104
|
end
|
|
94
105
|
|
|
106
|
+
def postgres_unready_payload(health)
|
|
107
|
+
{
|
|
108
|
+
status: "unready",
|
|
109
|
+
service: "mammoth",
|
|
110
|
+
name: mammoth_name,
|
|
111
|
+
operational_state: "ok",
|
|
112
|
+
adapter: state_summary.fetch(:adapter),
|
|
113
|
+
summary: state_summary,
|
|
114
|
+
postgres_slot: health.summary,
|
|
115
|
+
checked_at: checked_at
|
|
116
|
+
}
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def postgres_error_payload(error)
|
|
120
|
+
{
|
|
121
|
+
status: "unready",
|
|
122
|
+
service: "mammoth",
|
|
123
|
+
name: mammoth_name,
|
|
124
|
+
operational_state: "ok",
|
|
125
|
+
adapter: configured_adapter_name,
|
|
126
|
+
postgres_slot: {
|
|
127
|
+
ready: false,
|
|
128
|
+
reason: "inspection failed",
|
|
129
|
+
error_class: error.class.name,
|
|
130
|
+
error_message: error.message
|
|
131
|
+
},
|
|
132
|
+
checked_at: checked_at
|
|
133
|
+
}
|
|
134
|
+
end
|
|
135
|
+
|
|
95
136
|
def aggregate_metric_lines(checkpoint_store:, dead_letter_store:, delivered_store:)
|
|
96
137
|
[
|
|
97
138
|
metric_line("mammoth_up", 1),
|
|
@@ -136,6 +177,18 @@ module Mammoth
|
|
|
136
177
|
"#{(metric_headers + [metric_line("mammoth_up", 0)] + dispatch_metric_lines).join("\n")}\n"
|
|
137
178
|
end
|
|
138
179
|
|
|
180
|
+
def postgres_slot_metric_lines
|
|
181
|
+
return [] unless slot_health_provider
|
|
182
|
+
|
|
183
|
+
postgres_metric_lines(postgres_slot_health)
|
|
184
|
+
rescue ReplicationError
|
|
185
|
+
postgres_inspection_error_metric_lines
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def postgres_slot_health
|
|
189
|
+
slot_health_provider.slot_health
|
|
190
|
+
end
|
|
191
|
+
|
|
139
192
|
def state_summary
|
|
140
193
|
@state_summary ||= state_adapter.summary
|
|
141
194
|
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mammoth
|
|
4
|
+
# PostgreSQL slot-specific Prometheus formatting helpers.
|
|
5
|
+
module PostgresObservabilityMetrics
|
|
6
|
+
# Metric names and help text for PostgreSQL replication-slot gauges.
|
|
7
|
+
POSTGRES_METRICS = {
|
|
8
|
+
"mammoth_postgres_slot_inspection_up" => "1 when PostgreSQL slot inspection succeeds.",
|
|
9
|
+
"mammoth_postgres_slot_present" => "1 when the configured replication slot exists.",
|
|
10
|
+
"mammoth_postgres_slot_ready" => "1 when the configured slot is healthy and active.",
|
|
11
|
+
"mammoth_postgres_slot_active" => "1 when PostgreSQL reports the slot as active.",
|
|
12
|
+
"mammoth_postgres_slot_retained_wal_bytes" => "WAL retained from the slot restart LSN.",
|
|
13
|
+
"mammoth_postgres_slot_safe_wal_size_bytes" => "WAL bytes that may be written before the slot risks loss.",
|
|
14
|
+
"mammoth_postgres_slot_wal_status" => "Current PostgreSQL WAL retention status for the slot.",
|
|
15
|
+
"mammoth_postgres_slot_invalidated" => "1 when PostgreSQL reports invalidation or conflict.",
|
|
16
|
+
"mammoth_postgres_slot_inactive_since_timestamp_seconds" => "Unix timestamp when the slot became inactive.",
|
|
17
|
+
"mammoth_postgres_slot_restart_lsn_bytes" => "Numeric PostgreSQL restart LSN.",
|
|
18
|
+
"mammoth_postgres_slot_confirmed_flush_lsn_bytes" => "Numeric PostgreSQL confirmed flush LSN."
|
|
19
|
+
}.freeze
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def postgres_metric_headers
|
|
24
|
+
POSTGRES_METRICS.flat_map { |name, help| ["# HELP #{name} #{help}", "# TYPE #{name} gauge"] }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def postgres_metric_lines(health)
|
|
28
|
+
labels = { slot_name: health.slot_name }
|
|
29
|
+
lines = postgres_presence_metric_lines(health, labels:)
|
|
30
|
+
return lines unless health.present
|
|
31
|
+
|
|
32
|
+
lines + postgres_status_metric_lines(health, labels:) + postgres_optional_metric_lines(health, labels:)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def postgres_presence_metric_lines(health, labels:)
|
|
36
|
+
[
|
|
37
|
+
metric_line("mammoth_postgres_slot_inspection_up", 1, labels:),
|
|
38
|
+
metric_line("mammoth_postgres_slot_present", health.present ? 1 : 0, labels:)
|
|
39
|
+
]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def postgres_status_metric_lines(health, labels:)
|
|
43
|
+
[
|
|
44
|
+
metric_line("mammoth_postgres_slot_ready", health.ready? ? 1 : 0, labels:),
|
|
45
|
+
metric_line("mammoth_postgres_slot_active", health.active ? 1 : 0, labels:),
|
|
46
|
+
metric_line("mammoth_postgres_slot_wal_status", 1,
|
|
47
|
+
labels: labels.merge(wal_status: health.wal_status || "unknown")),
|
|
48
|
+
metric_line("mammoth_postgres_slot_invalidated", invalidated?(health) ? 1 : 0, labels:)
|
|
49
|
+
]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def postgres_optional_metric_lines(health, labels:)
|
|
53
|
+
{
|
|
54
|
+
"mammoth_postgres_slot_retained_wal_bytes" => health.retained_wal_bytes,
|
|
55
|
+
"mammoth_postgres_slot_safe_wal_size_bytes" => health.safe_wal_size,
|
|
56
|
+
"mammoth_postgres_slot_inactive_since_timestamp_seconds" => timestamp_seconds(health.inactive_since),
|
|
57
|
+
"mammoth_postgres_slot_restart_lsn_bytes" => health.restart_lsn_bytes,
|
|
58
|
+
"mammoth_postgres_slot_confirmed_flush_lsn_bytes" => health.confirmed_flush_lsn_bytes
|
|
59
|
+
}.filter_map { |name, value| metric_line(name, value, labels:) unless value.nil? }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def postgres_inspection_error_metric_lines
|
|
63
|
+
labels = { slot_name: config.dig("replication", "slot") }
|
|
64
|
+
[metric_line("mammoth_postgres_slot_inspection_up", 0, labels:)]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def timestamp_seconds(value)
|
|
68
|
+
return nil if value.nil?
|
|
69
|
+
return value.to_i if value.respond_to?(:to_i) && !value.is_a?(String)
|
|
70
|
+
|
|
71
|
+
Time.parse(value.to_s).to_i
|
|
72
|
+
rescue ArgumentError
|
|
73
|
+
nil
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def invalidated?(health)
|
|
77
|
+
health.conflicting || (!health.invalidation_reason.nil? && health.invalidation_reason != "")
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -27,7 +27,7 @@ module Mammoth
|
|
|
27
27
|
|
|
28
28
|
count = 0
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
each_grouped_event do |event, _group_end|
|
|
31
31
|
yield event
|
|
32
32
|
count += 1
|
|
33
33
|
end
|
|
@@ -35,11 +35,27 @@ module Mammoth
|
|
|
35
35
|
count
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
+
# Consume work with an explicit source-group boundary for safe progress.
|
|
39
|
+
#
|
|
40
|
+
# @yieldparam event [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] core work item
|
|
41
|
+
# @yieldparam group_end [Boolean] true for the final delivery in a source transaction
|
|
42
|
+
# @return [Integer] number of consumed work items
|
|
43
|
+
def start_with_boundaries
|
|
44
|
+
return enum_for(:start_with_boundaries) unless block_given?
|
|
45
|
+
|
|
46
|
+
count = 0
|
|
47
|
+
each_grouped_event do |event, group_end|
|
|
48
|
+
yield event, group_end
|
|
49
|
+
count += 1
|
|
50
|
+
end
|
|
51
|
+
count
|
|
52
|
+
end
|
|
53
|
+
|
|
38
54
|
private
|
|
39
55
|
|
|
40
|
-
def
|
|
56
|
+
def each_grouped_event(&block)
|
|
41
57
|
effective_source.each do |work|
|
|
42
|
-
|
|
58
|
+
grouped_cdc_work(work).each { |event, group_end| block.call(event, group_end) }
|
|
43
59
|
end
|
|
44
60
|
end
|
|
45
61
|
|
|
@@ -47,15 +63,20 @@ module Mammoth
|
|
|
47
63
|
source || raise(ReplicationError, "replication source is not configured")
|
|
48
64
|
end
|
|
49
65
|
|
|
50
|
-
def
|
|
66
|
+
def grouped_cdc_work(work)
|
|
51
67
|
return [] if work.nil?
|
|
52
|
-
return work.flat_map { |item|
|
|
53
|
-
return
|
|
54
|
-
return event_work(work) if work.is_a?(CDC::Core::ChangeEvent)
|
|
68
|
+
return work.flat_map { |item| grouped_cdc_work(item) } if work.is_a?(Array)
|
|
69
|
+
return grouped_transaction_work(work) if work.is_a?(CDC::Core::TransactionEnvelope)
|
|
70
|
+
return event_work(work).map { |item| [item, true] } if work.is_a?(CDC::Core::ChangeEvent)
|
|
55
71
|
|
|
56
72
|
raise ReplicationError, "CDC source yielded non-core work: #{work.class}"
|
|
57
73
|
end
|
|
58
74
|
|
|
75
|
+
def grouped_transaction_work(envelope)
|
|
76
|
+
items = transaction_work(envelope)
|
|
77
|
+
items.each_with_index.map { |item, index| [item, index == items.length - 1] }
|
|
78
|
+
end
|
|
79
|
+
|
|
59
80
|
def transaction_delivery?
|
|
60
81
|
delivery_unit == :transaction
|
|
61
82
|
end
|
|
@@ -27,6 +27,8 @@ module Mammoth
|
|
|
27
27
|
attr_reader :adapter
|
|
28
28
|
# @return [Mammoth::CheckpointStore, nil] checkpoint store used for restart resume
|
|
29
29
|
attr_reader :checkpoint_store
|
|
30
|
+
# @return [#inspect, nil] injected PostgreSQL publication inspector
|
|
31
|
+
attr_reader :publication_inspector
|
|
30
32
|
|
|
31
33
|
# Build a PostgreSQL CDC source.
|
|
32
34
|
#
|
|
@@ -36,13 +38,16 @@ module Mammoth
|
|
|
36
38
|
# @param decoder [Object, nil] injected pgoutput decoder
|
|
37
39
|
# @param adapter [Object, nil] injected source adapter
|
|
38
40
|
# @param checkpoint_store [Mammoth::CheckpointStore, nil] persisted checkpoints for restart resume
|
|
39
|
-
|
|
41
|
+
# @param publication_inspector [#inspect, nil] injected publication metadata inspector
|
|
42
|
+
def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil, checkpoint_store: nil,
|
|
43
|
+
publication_inspector: nil)
|
|
40
44
|
@config = config
|
|
41
45
|
@runner = runner
|
|
42
46
|
@parser = parser
|
|
43
47
|
@decoder = decoder
|
|
44
48
|
@adapter = adapter
|
|
45
49
|
@checkpoint_store = checkpoint_store
|
|
50
|
+
@publication_inspector = publication_inspector
|
|
46
51
|
end
|
|
47
52
|
|
|
48
53
|
# Stream CDC::Core work from PostgreSQL logical replication.
|
|
@@ -59,13 +64,15 @@ module Mammoth
|
|
|
59
64
|
def each(&block)
|
|
60
65
|
return enum_for(:each) unless block_given?
|
|
61
66
|
|
|
67
|
+
preflight_slot!
|
|
68
|
+
preflight_replica_identity!
|
|
62
69
|
normalizer = effective_adapter
|
|
63
70
|
unless normalizer.respond_to?(:each_normalized)
|
|
64
71
|
raise ReplicationError, "pgoutput source adapter must respond to #each_normalized"
|
|
65
72
|
end
|
|
66
73
|
|
|
67
74
|
normalizer.each_normalized(decoded_stream) do |work|
|
|
68
|
-
|
|
75
|
+
yield_with_progress_position(validate_core_work!(work), &block)
|
|
69
76
|
end
|
|
70
77
|
nil
|
|
71
78
|
rescue StandardError => e
|
|
@@ -74,11 +81,65 @@ module Mammoth
|
|
|
74
81
|
raise ReplicationError, "PostgreSQL CDC source failed: #{e.message}"
|
|
75
82
|
end
|
|
76
83
|
|
|
84
|
+
# Acknowledge a durably handled PostgreSQL WAL position.
|
|
85
|
+
#
|
|
86
|
+
# @param lsn [String, Integer] pgoutput-client compatible WAL position
|
|
87
|
+
# @return [Integer] normalized acknowledged WAL position
|
|
88
|
+
def acknowledge(lsn)
|
|
89
|
+
effective_runner.ack(lsn)
|
|
90
|
+
rescue StandardError => e
|
|
91
|
+
raise ReplicationError, "PostgreSQL WAL acknowledgement failed: #{e.message}"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Resolve the acknowledgement-compatible transport position for work
|
|
95
|
+
# currently being yielded by this source.
|
|
96
|
+
#
|
|
97
|
+
# @param work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] yielded work
|
|
98
|
+
# @return [String, Integer, nil] pgoutput-client compatible WAL position
|
|
99
|
+
def progress_position_for(work)
|
|
100
|
+
return nil unless yielded_work_includes?(work)
|
|
101
|
+
|
|
102
|
+
@yielded_progress_position
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Inspect the configured slot for readiness and operator metrics.
|
|
106
|
+
#
|
|
107
|
+
# pgoutput-client owns catalog access. This method converts its snapshot
|
|
108
|
+
# into Mammoth's PostgreSQL-specific health policy without leaking
|
|
109
|
+
# transport-library types into the observability layer.
|
|
110
|
+
#
|
|
111
|
+
# @return [PostgresSlotHealth]
|
|
112
|
+
def slot_health
|
|
113
|
+
status = inspected_slot_status
|
|
114
|
+
return PostgresSlotHealth.missing(required_config("replication", "slot")) unless status
|
|
115
|
+
|
|
116
|
+
PostgresSlotHealth.new(
|
|
117
|
+
slot_name: value_from(status, :slot_name),
|
|
118
|
+
present: true,
|
|
119
|
+
active: value_from(status, :active) == true,
|
|
120
|
+
retained_wal_bytes: value_from(status, :retained_wal_bytes),
|
|
121
|
+
wal_status: value_from(status, :wal_status),
|
|
122
|
+
safe_wal_size: value_from(status, :safe_wal_size),
|
|
123
|
+
inactive_since: value_from(status, :inactive_since),
|
|
124
|
+
invalidation_reason: value_from(status, :invalidation_reason),
|
|
125
|
+
restart_lsn: value_from(status, :restart_lsn),
|
|
126
|
+
restart_lsn_bytes: metric_lsn(value_from(status, :restart_lsn)),
|
|
127
|
+
confirmed_flush_lsn: value_from(status, :confirmed_flush_lsn),
|
|
128
|
+
confirmed_flush_lsn_bytes: metric_lsn(value_from(status, :confirmed_flush_lsn)),
|
|
129
|
+
conflicting: value_from(status, :conflicting) == true
|
|
130
|
+
)
|
|
131
|
+
rescue StandardError => e
|
|
132
|
+
raise e if e.is_a?(ReplicationError)
|
|
133
|
+
|
|
134
|
+
raise ReplicationError, "PostgreSQL slot health inspection failed: #{e.message}"
|
|
135
|
+
end
|
|
136
|
+
|
|
77
137
|
private
|
|
78
138
|
|
|
79
139
|
def decoded_stream
|
|
80
140
|
Enumerator.new do |stream|
|
|
81
141
|
effective_runner.start do |payload, metadata = nil|
|
|
142
|
+
@latest_progress_position = acknowledgement_position(metadata)
|
|
82
143
|
process_payload(payload, metadata) { |decoded| stream << stream_event(decoded, metadata) }
|
|
83
144
|
end
|
|
84
145
|
end
|
|
@@ -108,6 +169,15 @@ module Mammoth
|
|
|
108
169
|
normalizer.stream_event(decoded, source_position: source_position(metadata))
|
|
109
170
|
end
|
|
110
171
|
|
|
172
|
+
def yield_with_progress_position(work)
|
|
173
|
+
@yielded_work = work
|
|
174
|
+
@yielded_progress_position = @latest_progress_position
|
|
175
|
+
yield work
|
|
176
|
+
ensure
|
|
177
|
+
@yielded_work = nil
|
|
178
|
+
@yielded_progress_position = nil
|
|
179
|
+
end
|
|
180
|
+
|
|
111
181
|
def validate_core_work!(work)
|
|
112
182
|
return work if work.is_a?(CDC::Core::ChangeEvent)
|
|
113
183
|
return work if work.is_a?(CDC::Core::TransactionEnvelope)
|
|
@@ -156,6 +226,22 @@ module Mammoth
|
|
|
156
226
|
value_from(metadata, :source_position, :commit_lsn, :lsn, :wal_end_lsn, :end_lsn, :final_lsn)
|
|
157
227
|
end
|
|
158
228
|
|
|
229
|
+
def acknowledgement_position(metadata)
|
|
230
|
+
value = value_from(metadata, :wal_end_lsn, :wal_end, :lsn)
|
|
231
|
+
return nil if value.nil?
|
|
232
|
+
return value if value.is_a?(Integer) && value >= 0
|
|
233
|
+
return value if value.is_a?(String) && value.match?(%r{\A[0-9A-F]+/[0-9A-F]+\z}i)
|
|
234
|
+
|
|
235
|
+
raise ReplicationError, "invalid PostgreSQL transport LSN for acknowledgement: #{value.inspect}"
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def yielded_work_includes?(work)
|
|
239
|
+
return true if @yielded_work.equal?(work)
|
|
240
|
+
return false unless @yielded_work.is_a?(CDC::Core::TransactionEnvelope)
|
|
241
|
+
|
|
242
|
+
@yielded_work.events.any? { |event| event.equal?(work) }
|
|
243
|
+
end
|
|
244
|
+
|
|
159
245
|
def effective_runner
|
|
160
246
|
runner || @effective_runner || begin
|
|
161
247
|
@effective_runner = build_runner
|
|
@@ -180,6 +266,12 @@ module Mammoth
|
|
|
180
266
|
end
|
|
181
267
|
end
|
|
182
268
|
|
|
269
|
+
def effective_publication_inspector
|
|
270
|
+
publication_inspector || @effective_publication_inspector || begin
|
|
271
|
+
@effective_publication_inspector = build_publication_inspector
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
183
275
|
def build_runner
|
|
184
276
|
require_optional!("pgoutput/client", "pgoutput-client")
|
|
185
277
|
|
|
@@ -201,7 +293,12 @@ module Mammoth
|
|
|
201
293
|
def build_adapter
|
|
202
294
|
require_optional!("pgoutput/source_adapter", "pgoutput-source-adapter")
|
|
203
295
|
|
|
204
|
-
Pgoutput::SourceAdapter::
|
|
296
|
+
resolver = Pgoutput::SourceAdapter::ReplicaIdentityResolver.new(replica_identity_relations)
|
|
297
|
+
Pgoutput::SourceAdapter::Cdc.new(primary_key_resolver: resolver)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def build_publication_inspector
|
|
301
|
+
PostgresPublicationInspector.new(database_url:)
|
|
205
302
|
end
|
|
206
303
|
|
|
207
304
|
def runner_options
|
|
@@ -210,7 +307,7 @@ module Mammoth
|
|
|
210
307
|
slot_name: required_config("replication", "slot"),
|
|
211
308
|
publication_names: required_publications,
|
|
212
309
|
start_lsn: replication_start_lsn,
|
|
213
|
-
auto_create_slot:
|
|
310
|
+
auto_create_slot: safe_auto_create_slot?,
|
|
214
311
|
temporary_slot: config.dig("replication", "temporary_slot") == true
|
|
215
312
|
}.tap do |options|
|
|
216
313
|
feedback_interval = config.dig("replication", "feedback_interval")
|
|
@@ -225,6 +322,157 @@ module Mammoth
|
|
|
225
322
|
checkpoint_lsn
|
|
226
323
|
end
|
|
227
324
|
|
|
325
|
+
def preflight_slot!
|
|
326
|
+
resume_lsn = replication_start_lsn
|
|
327
|
+
validate_temporary_slot_resume!(resume_lsn)
|
|
328
|
+
status = inspected_slot_status
|
|
329
|
+
|
|
330
|
+
if status.nil?
|
|
331
|
+
return if safe_auto_create_slot?
|
|
332
|
+
|
|
333
|
+
raise ReplicationError, missing_slot_message(resume_lsn)
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
validate_slot_identity!(status)
|
|
337
|
+
validate_slot_health!(status)
|
|
338
|
+
validate_slot_reachability!(status, resume_lsn) unless blank?(resume_lsn)
|
|
339
|
+
nil
|
|
340
|
+
rescue StandardError => e
|
|
341
|
+
raise e if e.is_a?(ReplicationError)
|
|
342
|
+
|
|
343
|
+
raise ReplicationError, "PostgreSQL slot preflight failed: #{e.message}"
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
def preflight_replica_identity!
|
|
347
|
+
invalid_tables = publication_tables.reject(&:identity_usable?)
|
|
348
|
+
return if invalid_tables.empty?
|
|
349
|
+
|
|
350
|
+
details = invalid_tables.map do |table|
|
|
351
|
+
"#{table.qualified_name} (actions=#{table.identity_actions.join("/")}, " \
|
|
352
|
+
"replica_identity=#{replica_identity_name(table.replica_identity)})"
|
|
353
|
+
end
|
|
354
|
+
raise ReplicationError,
|
|
355
|
+
"PostgreSQL replica identity preflight failed for #{details.join(", ")}. " \
|
|
356
|
+
"Add a primary key, select an eligible unique index with REPLICA IDENTITY USING INDEX, " \
|
|
357
|
+
"use REPLICA IDENTITY FULL, or remove UPDATE/DELETE from the publication."
|
|
358
|
+
rescue StandardError => e
|
|
359
|
+
raise e if e.is_a?(ReplicationError)
|
|
360
|
+
|
|
361
|
+
raise ReplicationError, "PostgreSQL replica identity preflight failed: #{e.message}"
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
def publication_tables
|
|
365
|
+
@publication_tables ||= effective_publication_inspector.inspect(required_publications).freeze
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def replica_identity_relations
|
|
369
|
+
relations = {} # : Hash[Integer | [String, String], Array[String]]
|
|
370
|
+
publication_tables.each do |table|
|
|
371
|
+
next if table.replica_identity_columns.empty?
|
|
372
|
+
|
|
373
|
+
columns = table.replica_identity_columns
|
|
374
|
+
relations[table.relation_id] = columns
|
|
375
|
+
relations[[table.schema_name, table.table_name]] = columns
|
|
376
|
+
end
|
|
377
|
+
relations
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
def inspected_slot_status
|
|
381
|
+
client = effective_runner
|
|
382
|
+
unless client.respond_to?(:slot_status)
|
|
383
|
+
raise ReplicationError, "pgoutput-client 0.4+ with #slot_status is required for PostgreSQL slot inspection"
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
client.slot_status
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
def validate_temporary_slot_resume!(resume_lsn)
|
|
390
|
+
return if blank?(resume_lsn)
|
|
391
|
+
return unless config.dig("replication", "temporary_slot") == true
|
|
392
|
+
|
|
393
|
+
raise ReplicationError, "cannot resume durable PostgreSQL checkpoint #{resume_lsn} with a temporary slot"
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def validate_slot_identity!(status)
|
|
397
|
+
slot_name = required_config("replication", "slot")
|
|
398
|
+
unless value_from(status, :slot_name) == slot_name
|
|
399
|
+
raise ReplicationError, "PostgreSQL slot preflight returned the wrong slot for #{slot_name}"
|
|
400
|
+
end
|
|
401
|
+
unless value_from(status, :slot_type) == "logical" && value_from(status, :plugin) == "pgoutput"
|
|
402
|
+
raise ReplicationError, "PostgreSQL slot #{slot_name} must be a logical pgoutput slot"
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
database = required_config("postgres", "database")
|
|
406
|
+
return if value_from(status, :database) == database
|
|
407
|
+
|
|
408
|
+
raise ReplicationError, "PostgreSQL slot #{slot_name} belongs to a different database"
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
def validate_slot_health!(status)
|
|
412
|
+
slot_name = required_config("replication", "slot")
|
|
413
|
+
raise ReplicationError, "PostgreSQL slot #{slot_name} is already active" if value_from(status, :active) == true
|
|
414
|
+
|
|
415
|
+
wal_status = value_from(status, :wal_status)
|
|
416
|
+
if %w[lost unreserved].include?(wal_status)
|
|
417
|
+
raise ReplicationError, "PostgreSQL slot #{slot_name} cannot retain required WAL (wal_status=#{wal_status})"
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
invalidation_reason = value_from(status, :invalidation_reason)
|
|
421
|
+
if value_from(status, :conflicting) == true || !blank?(invalidation_reason)
|
|
422
|
+
raise ReplicationError,
|
|
423
|
+
"PostgreSQL slot #{slot_name} is invalidated#{": #{invalidation_reason}" unless blank?(invalidation_reason)}"
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
return unless blank?(value_from(status, :restart_lsn))
|
|
427
|
+
|
|
428
|
+
raise ReplicationError, "PostgreSQL slot #{slot_name} has no reachable restart LSN"
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
def validate_slot_reachability!(status, resume_lsn)
|
|
432
|
+
requested = parse_transport_lsn(resume_lsn, "resume checkpoint")
|
|
433
|
+
%i[restart_lsn confirmed_flush_lsn].each do |field|
|
|
434
|
+
boundary = value_from(status, field)
|
|
435
|
+
next if blank?(boundary)
|
|
436
|
+
next unless requested < parse_transport_lsn(boundary, field.to_s)
|
|
437
|
+
|
|
438
|
+
raise ReplicationError,
|
|
439
|
+
"PostgreSQL slot cannot serve checkpoint #{resume_lsn}; #{field}=#{boundary} has already advanced past it"
|
|
440
|
+
end
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
def parse_transport_lsn(value, label)
|
|
444
|
+
require_optional!("pgoutput/client", "pgoutput-client")
|
|
445
|
+
Pgoutput::Client::LSN.parse(value)
|
|
446
|
+
rescue StandardError => e
|
|
447
|
+
raise ReplicationError, "invalid PostgreSQL #{label} LSN #{value.inspect}: #{e.message}"
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
def metric_lsn(value)
|
|
451
|
+
return nil if blank?(value)
|
|
452
|
+
|
|
453
|
+
parse_transport_lsn(value, "slot metric")
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def replica_identity_name(value)
|
|
457
|
+
{
|
|
458
|
+
"d" => "default",
|
|
459
|
+
"n" => "nothing",
|
|
460
|
+
"f" => "full",
|
|
461
|
+
"i" => "index"
|
|
462
|
+
}.fetch(value, value)
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
def safe_auto_create_slot?
|
|
466
|
+
config.dig("replication", "auto_create_slot") == true && blank?(replication_start_lsn)
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
def missing_slot_message(resume_lsn)
|
|
470
|
+
slot_name = required_config("replication", "slot")
|
|
471
|
+
return "PostgreSQL slot #{slot_name} is missing and auto_create_slot is disabled" if blank?(resume_lsn)
|
|
472
|
+
|
|
473
|
+
"PostgreSQL slot #{slot_name} is missing; refusing to recreate it while checkpoint #{resume_lsn} requires continuity"
|
|
474
|
+
end
|
|
475
|
+
|
|
228
476
|
def checkpoint_lsn
|
|
229
477
|
return nil unless checkpoint_store
|
|
230
478
|
|