mammoth 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +28 -0
- data/README.md +30 -2
- data/config/mammoth.example.yml +7 -2
- data/lib/mammoth/application.rb +32 -5
- data/lib/mammoth/cli.rb +1 -1
- data/lib/mammoth/delivery_progress_coordinator.rb +5 -2
- data/lib/mammoth/delivery_worker.rb +32 -11
- data/lib/mammoth/logging.rb +60 -0
- data/lib/mammoth/observability_server.rb +11 -2
- data/lib/mammoth/sources/postgres.rb +16 -3
- data/lib/mammoth/version.rb +1 -1
- data/lib/mammoth.rb +1 -0
- metadata +22 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7d16f1b05d2ae82569933033416a20892ae7e7afc9cdd282aacd6343789157c3
|
|
4
|
+
data.tar.gz: 1af59d9757dd584f462c5a888246c73693459aff503f6ef775e24aa4c4d19b49
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 14452ab3df0aaae412be115a33b25b20ae3ef9698146e2f0e00c38f0453245ea9abec2a1cdd93a6f780288e722fb982b3fa3f7ab7fafc0de979c19abccd9a74e
|
|
7
|
+
data.tar.gz: 39832167b4069a9b8b8b4bab5ad86f2a3a150449e216d41a4a915fc2e748b471d3361f183b088a7a990402515351d6bf7e3e19c5576a683dabdb463e6e508dce
|
data/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,34 @@
|
|
|
4
4
|
|
|
5
5
|
# Changelog
|
|
6
6
|
|
|
7
|
+
## Unreleased
|
|
8
|
+
|
|
9
|
+
## 1.5.0 - 2026-07-19
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- Default the unauthenticated observability HTTP server to the loopback
|
|
14
|
+
interface and document the network controls required for non-loopback
|
|
15
|
+
bindings.
|
|
16
|
+
- Implement `logging.level` with container-friendly JSON stdout logs for
|
|
17
|
+
application and replication lifecycle, delivery outcomes, retries, dead
|
|
18
|
+
letters, checkpoints, and debug-level work progress.
|
|
19
|
+
- Require a patched `json` gem and remove the vulnerable bundled default-gem
|
|
20
|
+
specification from the runtime image to address CVE-2026-33210.
|
|
21
|
+
|
|
22
|
+
### Quality
|
|
23
|
+
|
|
24
|
+
- Run the container quickstart smoke test on pull requests and whenever Docker,
|
|
25
|
+
application, configuration, or dependency inputs change.
|
|
26
|
+
- Added Dependabot updates, Ruby dependency auditing, dependency review,
|
|
27
|
+
CodeQL analysis, and high- and critical-severity container vulnerability
|
|
28
|
+
scanning.
|
|
29
|
+
- Pin the Ruby container base image to an immutable digest and publish release
|
|
30
|
+
images with SBOM and maximum provenance attestations.
|
|
31
|
+
- Published a full reference benchmark snapshot with machine-readable results,
|
|
32
|
+
raw output, and documented interpretation across projection, delivery,
|
|
33
|
+
fanout, SQLite operational state, observability, and dead-letter replay.
|
|
34
|
+
|
|
7
35
|
## 1.4.0 - 2026-07-19
|
|
8
36
|
|
|
9
37
|
### Added
|
data/README.md
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
[](https://rubygems.org/gems/mammoth)
|
|
18
18
|
[](https://www.ruby-lang.org/)
|
|
19
19
|
[](https://github.com/kanutocd/mammoth/actions/workflows/ci.yml)
|
|
20
|
+
[](https://github.com/kanutocd/mammoth/actions/workflows/security.yml)
|
|
20
21
|
[](https://github.com/kanutocd/mammoth/blob/main/.github/workflows/ci.yml#L50)
|
|
21
22
|
[](LICENSE.txt)
|
|
22
23
|
|
|
@@ -106,6 +107,7 @@ Mammoth 1.x includes:
|
|
|
106
107
|
- per-destination payload removal and masking policies
|
|
107
108
|
- per-destination enable/disable and retry policy controls
|
|
108
109
|
- delivery worker with retry, delivered-ledger, and DLQ handling
|
|
110
|
+
- configurable `debug`, `info`, `warn`, and `error` JSON logging to stdout
|
|
109
111
|
- contiguous delivery watermark for checkpoint and PostgreSQL acknowledgement
|
|
110
112
|
- source-owned transport LSN preservation independent of payload `commit_lsn`
|
|
111
113
|
- fail-closed PostgreSQL slot and checkpoint continuity preflight
|
|
@@ -257,6 +259,30 @@ bundle exec ./exe/mammoth start config/mammoth.example.yml
|
|
|
257
259
|
bundle exec ./exe/mammoth observability config/mammoth.example.yml
|
|
258
260
|
```
|
|
259
261
|
|
|
262
|
+
## Logging
|
|
263
|
+
|
|
264
|
+
Mammoth emits one JSON object per line to standard output. Set
|
|
265
|
+
`logging.level` to `debug`, `info`, `warn`, or `error`; `info` is the
|
|
266
|
+
recommended normal setting, while `debug` adds per-work and WAL acknowledgement
|
|
267
|
+
records.
|
|
268
|
+
|
|
269
|
+
```yaml
|
|
270
|
+
logging:
|
|
271
|
+
level: info
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
In Docker Compose, follow the application, replication, delivery, retry,
|
|
275
|
+
dead-letter, and checkpoint activity with:
|
|
276
|
+
|
|
277
|
+
```bash
|
|
278
|
+
docker compose logs -f mammoth
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Operational identifiers are included, but payload bodies, configured headers,
|
|
282
|
+
credentials, signing secrets, and exception messages are excluded. See
|
|
283
|
+
[`docs/CONFIGURATION.md`](docs/CONFIGURATION.md#logging) for the event and level
|
|
284
|
+
details.
|
|
285
|
+
|
|
260
286
|
Reconstruct and deliver a single persisted event JSON file through Mammoth's
|
|
261
287
|
core delivery path:
|
|
262
288
|
|
|
@@ -292,8 +318,10 @@ production:
|
|
|
292
318
|
- observability snapshots
|
|
293
319
|
- DLQ replay
|
|
294
320
|
|
|
295
|
-
The
|
|
296
|
-
|
|
321
|
+
The [reference snapshot](https://github.com/kanutocd/mammoth/blob/main/benchmark/results/20260724T104950Z/snapshot.md)
|
|
322
|
+
records a full local run, its exact commands, environment, raw output, and
|
|
323
|
+
interpretation. It is a reproducible baseline, not a universal performance
|
|
324
|
+
claim. Re-run the scripts in
|
|
297
325
|
[`benchmark/`](https://github.com/kanutocd/mammoth/tree/main/benchmark) on your own hardware when choosing
|
|
298
326
|
`runtime.concurrency`, `destinations`, SQLite storage, scrape frequency, and
|
|
299
327
|
DLQ replay expectations.
|
data/config/mammoth.example.yml
CHANGED
|
@@ -385,7 +385,11 @@ operational_state:
|
|
|
385
385
|
|
|
386
386
|
observability:
|
|
387
387
|
# Bind host for the optional health, readiness, and metrics server.
|
|
388
|
-
|
|
388
|
+
#
|
|
389
|
+
# The endpoints do not authenticate clients and may expose operational
|
|
390
|
+
# details. Keep the loopback default unless a trusted network boundary or
|
|
391
|
+
# authenticated reverse proxy protects the listener.
|
|
392
|
+
host: 127.0.0.1
|
|
389
393
|
|
|
390
394
|
# HTTP port for /healthz, /readyz, and /metrics.
|
|
391
395
|
port: 9393
|
|
@@ -397,7 +401,8 @@ logging:
|
|
|
397
401
|
# Very chatty; useful while diagnosing delivery or replication behavior.
|
|
398
402
|
#
|
|
399
403
|
# info:
|
|
400
|
-
# Recommended normal default.
|
|
404
|
+
# Recommended normal default. Emits lifecycle, delivery, checkpoint, retry,
|
|
405
|
+
# and dead-letter JSON records to stdout for container log collection.
|
|
401
406
|
#
|
|
402
407
|
# warn/error:
|
|
403
408
|
# Quieter production modes.
|
data/lib/mammoth/application.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Mammoth
|
|
|
12
12
|
# rubocop:disable Metrics/ClassLength
|
|
13
13
|
class Application
|
|
14
14
|
attr_reader :config, :state_adapter, :consumer, :delivery_worker, :checkpoint_store, :lifecycle_hooks, :observer,
|
|
15
|
-
:progress_coordinator
|
|
15
|
+
:progress_coordinator, :logger
|
|
16
16
|
|
|
17
17
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
18
18
|
# @param source [#each, nil] injectable event source for tests and demos
|
|
@@ -21,9 +21,11 @@ module Mammoth
|
|
|
21
21
|
# @param sleeper [#call] retry sleep strategy
|
|
22
22
|
# @param lifecycle_hooks [Mammoth::LifecycleHooks, Hash] local lifecycle callbacks
|
|
23
23
|
# @param observer [CDC::Core::Observer, nil] dispatch lifecycle observer
|
|
24
|
+
# @param logger [Mammoth::Logging::Logger, nil] structured operational logger
|
|
24
25
|
def initialize(config, source: nil, sink: nil, state_adapter: nil, sleeper: Kernel.method(:sleep),
|
|
25
|
-
lifecycle_hooks: LifecycleHooks.new, observer: nil)
|
|
26
|
+
lifecycle_hooks: LifecycleHooks.new, observer: nil, logger: nil)
|
|
26
27
|
@config = config
|
|
28
|
+
@logger = logger || Logging.build(config)
|
|
27
29
|
@lifecycle_hooks = build_lifecycle_hooks(lifecycle_hooks)
|
|
28
30
|
@observer = observer || MetricsObserver.new
|
|
29
31
|
@state_adapter = state_adapter || build_state_adapter
|
|
@@ -49,22 +51,37 @@ module Mammoth
|
|
|
49
51
|
# Start the application runtime and deliver consumed CDC work.
|
|
50
52
|
#
|
|
51
53
|
# @return [Integer] number of processed work units
|
|
54
|
+
# rubocop:disable Metrics/AbcSize
|
|
52
55
|
def start
|
|
53
56
|
runtime = build_runtime
|
|
54
57
|
processed = nil
|
|
55
58
|
|
|
59
|
+
operational_logger.info("application_started", mammoth_name: config&.dig("mammoth", "name"),
|
|
60
|
+
runtime: config&.dig("runtime", "adapter"),
|
|
61
|
+
delivery_unit: config&.dig("delivery", "unit"))
|
|
56
62
|
lifecycle_hooks.call(:before_start, application_context(runtime: runtime))
|
|
57
63
|
processed = process_consumer(runtime)
|
|
58
64
|
lifecycle_hooks.call(:after_start, application_context(runtime: runtime, processed: processed))
|
|
65
|
+
operational_logger.info("application_completed", mammoth_name: config&.dig("mammoth", "name"), processed:)
|
|
59
66
|
processed
|
|
67
|
+
rescue StandardError => e
|
|
68
|
+
operational_logger.error("application_failed", mammoth_name: config&.dig("mammoth", "name"),
|
|
69
|
+
error_class: e.class.name)
|
|
70
|
+
raise
|
|
60
71
|
ensure
|
|
61
72
|
lifecycle_hooks.call(:before_shutdown, application_context(runtime: runtime, processed: processed))
|
|
62
73
|
runtime.shutdown if runtime.respond_to?(:shutdown)
|
|
63
74
|
lifecycle_hooks.call(:after_shutdown, application_context(runtime: runtime, processed: processed))
|
|
75
|
+
operational_logger.info("application_stopped", mammoth_name: config&.dig("mammoth", "name"), processed:)
|
|
64
76
|
end
|
|
77
|
+
# rubocop:enable Metrics/AbcSize
|
|
65
78
|
|
|
66
79
|
private
|
|
67
80
|
|
|
81
|
+
def operational_logger
|
|
82
|
+
logger || Logging::NullLogger::INSTANCE
|
|
83
|
+
end
|
|
84
|
+
|
|
68
85
|
def build_lifecycle_hooks(hooks)
|
|
69
86
|
return hooks if hooks.is_a?(LifecycleHooks)
|
|
70
87
|
|
|
@@ -84,6 +101,7 @@ module Mammoth
|
|
|
84
101
|
processed = 0
|
|
85
102
|
|
|
86
103
|
consumer.start_with_boundaries do |work, group_end|
|
|
104
|
+
operational_logger.debug("work_received", work_type: work.class.name, work_id: work_identifier(work), group_end:)
|
|
87
105
|
progress_coordinator.register(work, group_end:)
|
|
88
106
|
process_work(runtime, work)
|
|
89
107
|
processed += 1
|
|
@@ -94,6 +112,13 @@ module Mammoth
|
|
|
94
112
|
processed
|
|
95
113
|
end
|
|
96
114
|
|
|
115
|
+
def work_identifier(work)
|
|
116
|
+
return work.transaction_id if work.respond_to?(:transaction_id)
|
|
117
|
+
return work.event_id if work.respond_to?(:event_id)
|
|
118
|
+
|
|
119
|
+
nil
|
|
120
|
+
end
|
|
121
|
+
|
|
97
122
|
def process_work(runtime, work)
|
|
98
123
|
runtime.process(work)
|
|
99
124
|
end
|
|
@@ -115,7 +140,7 @@ module Mammoth
|
|
|
115
140
|
end
|
|
116
141
|
|
|
117
142
|
def build_source
|
|
118
|
-
Sources::Postgres.new(config, checkpoint_store: checkpoint_store)
|
|
143
|
+
Sources::Postgres.new(config, checkpoint_store: checkpoint_store, logger: logger)
|
|
119
144
|
end
|
|
120
145
|
|
|
121
146
|
def build_progress_coordinator
|
|
@@ -125,7 +150,8 @@ module Mammoth
|
|
|
125
150
|
slot_name: config.dig("replication", "slot"),
|
|
126
151
|
publication_name: Array(config.dig("replication", "publications")).join(","),
|
|
127
152
|
acknowledger: source_acknowledger,
|
|
128
|
-
position_resolver: source_position_resolver
|
|
153
|
+
position_resolver: source_position_resolver,
|
|
154
|
+
logger: logger
|
|
129
155
|
)
|
|
130
156
|
end
|
|
131
157
|
|
|
@@ -147,7 +173,8 @@ module Mammoth
|
|
|
147
173
|
dead_letter_store: state_adapter.dead_letter_store,
|
|
148
174
|
delivered_envelope_store: state_adapter.delivered_envelope_store,
|
|
149
175
|
sleeper: sleeper,
|
|
150
|
-
delivery_policy: delivery_policy
|
|
176
|
+
delivery_policy: delivery_policy,
|
|
177
|
+
logger: logger
|
|
151
178
|
)
|
|
152
179
|
end
|
|
153
180
|
|
data/lib/mammoth/cli.rb
CHANGED
|
@@ -109,7 +109,7 @@ module Mammoth
|
|
|
109
109
|
def observability
|
|
110
110
|
config = load_config
|
|
111
111
|
server = ObservabilityServer.new(config)
|
|
112
|
-
|
|
112
|
+
Logging.build(config).info("observability_started", host: server.host, port: server.port)
|
|
113
113
|
server.start
|
|
114
114
|
0
|
|
115
115
|
end
|
|
@@ -36,7 +36,7 @@ module Mammoth
|
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
attr_reader :checkpoint_store, :source_name, :slot_name, :publication_name
|
|
39
|
+
attr_reader :checkpoint_store, :source_name, :slot_name, :publication_name, :logger
|
|
40
40
|
|
|
41
41
|
# @param checkpoint_store [Mammoth::CheckpointStore] durable checkpoint persistence
|
|
42
42
|
# @param source_name [String] logical source name
|
|
@@ -45,13 +45,14 @@ module Mammoth
|
|
|
45
45
|
# @param acknowledger [#call, nil] upstream durable-progress acknowledgement
|
|
46
46
|
# @param position_resolver [#call, nil] source-owned durable position resolver
|
|
47
47
|
def initialize(checkpoint_store:, source_name:, slot_name:, publication_name:, acknowledger: nil,
|
|
48
|
-
position_resolver: nil)
|
|
48
|
+
position_resolver: nil, logger: Logging::NullLogger::INSTANCE)
|
|
49
49
|
@checkpoint_store = checkpoint_store
|
|
50
50
|
@source_name = source_name
|
|
51
51
|
@slot_name = slot_name
|
|
52
52
|
@publication_name = publication_name
|
|
53
53
|
@acknowledger = acknowledger
|
|
54
54
|
@position_resolver = position_resolver
|
|
55
|
+
@logger = logger
|
|
55
56
|
# rubocop:disable Layout/LeadingCommentSpace -- Steep inline type syntax requires `#:`.
|
|
56
57
|
@groups = [] #: Array[Group]
|
|
57
58
|
@entries_by_work = {} #: Hash[untyped, Array[Entry]]
|
|
@@ -133,6 +134,8 @@ module Mammoth
|
|
|
133
134
|
last_lsn: position
|
|
134
135
|
)
|
|
135
136
|
@acknowledger&.call(position)
|
|
137
|
+
logger.info("checkpoint_advanced", mammoth_name: source_name, slot: slot_name, publication: publication_name,
|
|
138
|
+
source_position: position)
|
|
136
139
|
end
|
|
137
140
|
|
|
138
141
|
def remove_group(group)
|
|
@@ -7,12 +7,12 @@ module Mammoth
|
|
|
7
7
|
# the delivery contract small: attempt webhook delivery, record idempotent
|
|
8
8
|
# success, and persist exhausted failures to the dead letter queue. Contiguous
|
|
9
9
|
# checkpointing is owned by DeliveryProgressCoordinator.
|
|
10
|
-
class DeliveryWorker
|
|
10
|
+
class DeliveryWorker # rubocop:disable Metrics/ClassLength
|
|
11
11
|
# Default source name used when an event does not provide one.
|
|
12
12
|
DEFAULT_SOURCE = "postgresql"
|
|
13
13
|
|
|
14
14
|
attr_reader :sink, :checkpoint_store, :dead_letter_store, :delivered_envelope_store, :retry_schedule, :max_attempts,
|
|
15
|
-
:sleeper, :source_name, :slot_name, :publication_name, :route_filter, :payload_policy, :enabled
|
|
15
|
+
:sleeper, :source_name, :slot_name, :publication_name, :route_filter, :payload_policy, :enabled, :logger
|
|
16
16
|
|
|
17
17
|
# @param sink [#deliver] destination sink
|
|
18
18
|
# @param checkpoint_store [Mammoth::CheckpointStore] retained dependency; shared progress owns writes
|
|
@@ -29,7 +29,7 @@ module Mammoth
|
|
|
29
29
|
# @param enabled [Boolean] whether this destination accepts new deliveries
|
|
30
30
|
def initialize(sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:, source_name:, slot_name:,
|
|
31
31
|
publication_name:, max_attempts:, retry_schedule:, sleeper: Kernel.method(:sleep),
|
|
32
|
-
route_filter: nil, payload_policy: nil, enabled: true)
|
|
32
|
+
route_filter: nil, payload_policy: nil, enabled: true, logger: Logging::NullLogger::INSTANCE)
|
|
33
33
|
@sink = sink
|
|
34
34
|
@checkpoint_store = checkpoint_store
|
|
35
35
|
@dead_letter_store = dead_letter_store
|
|
@@ -43,6 +43,7 @@ module Mammoth
|
|
|
43
43
|
@route_filter = route_filter || RouteFilter.new
|
|
44
44
|
@payload_policy = payload_policy || PayloadPolicy.new
|
|
45
45
|
@enabled = enabled
|
|
46
|
+
@logger = logger
|
|
46
47
|
return unless @payload_policy.active? && !sink.respond_to?(:deliver_payload)
|
|
47
48
|
|
|
48
49
|
raise ConfigurationError, "destination #{destination_name} does not accept prepared payloads"
|
|
@@ -58,7 +59,7 @@ module Mammoth
|
|
|
58
59
|
# @param sleeper [#call] sleep strategy
|
|
59
60
|
# @return [Mammoth::DeliveryWorker]
|
|
60
61
|
def self.from_config(config, sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:,
|
|
61
|
-
sleeper: Kernel.method(:sleep), delivery_policy: {})
|
|
62
|
+
sleeper: Kernel.method(:sleep), delivery_policy: {}, logger: Logging::NullLogger::INSTANCE)
|
|
62
63
|
new(
|
|
63
64
|
sink: sink,
|
|
64
65
|
checkpoint_store: checkpoint_store,
|
|
@@ -72,7 +73,8 @@ module Mammoth
|
|
|
72
73
|
sleeper: sleeper,
|
|
73
74
|
route_filter: delivery_policy.fetch("route_filter", RouteFilter.new),
|
|
74
75
|
payload_policy: delivery_policy.fetch("payload_policy", PayloadPolicy.new),
|
|
75
|
-
enabled: delivery_policy.fetch("enabled", true)
|
|
76
|
+
enabled: delivery_policy.fetch("enabled", true),
|
|
77
|
+
logger: logger
|
|
76
78
|
)
|
|
77
79
|
end
|
|
78
80
|
|
|
@@ -109,7 +111,7 @@ module Mammoth
|
|
|
109
111
|
deliver_prepared(prepared, work: work, delivery_method: delivery_method)
|
|
110
112
|
end
|
|
111
113
|
|
|
112
|
-
# rubocop:disable Metrics/MethodLength
|
|
114
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
113
115
|
def deliver_prepared(prepared, work:, delivery_method:)
|
|
114
116
|
attempts = 0
|
|
115
117
|
canonical_payload = prepared.canonical_payload
|
|
@@ -118,16 +120,21 @@ module Mammoth
|
|
|
118
120
|
idempotency_key = idempotency_key_for(payload: canonical_payload, delivery_unit: delivery_unit)
|
|
119
121
|
|
|
120
122
|
skip_result = skip_result_for(canonical_payload, idempotency_key: idempotency_key)
|
|
121
|
-
|
|
123
|
+
if skip_result
|
|
124
|
+
log_outcome("delivery_skipped", skip_result, delivery_unit:)
|
|
125
|
+
return skip_result
|
|
126
|
+
end
|
|
122
127
|
|
|
123
128
|
if delivered_envelope_store.delivered?(idempotency_key)
|
|
124
|
-
|
|
129
|
+
result = {
|
|
125
130
|
status: "skipped",
|
|
126
131
|
duplicate: true,
|
|
127
132
|
idempotency_key: idempotency_key,
|
|
128
133
|
attempts: attempts,
|
|
129
134
|
destination: destination_name
|
|
130
135
|
}
|
|
136
|
+
log_outcome("delivery_skipped", result, delivery_unit:)
|
|
137
|
+
return result
|
|
131
138
|
end
|
|
132
139
|
|
|
133
140
|
begin
|
|
@@ -142,15 +149,19 @@ module Mammoth
|
|
|
142
149
|
transaction_id: payload["transaction_id"],
|
|
143
150
|
source_position: payload["source_position"]
|
|
144
151
|
)
|
|
145
|
-
result.merge(attempts: attempts, idempotency_key: idempotency_key)
|
|
152
|
+
result = result.merge(attempts: attempts, idempotency_key: idempotency_key)
|
|
153
|
+
log_outcome("delivery_succeeded", result, delivery_unit:)
|
|
154
|
+
result
|
|
146
155
|
rescue DeliveryError => e
|
|
147
156
|
return dead_letter(payload, e, attempts) if attempts >= max_attempts
|
|
148
157
|
|
|
149
|
-
wait_before_retry(attempts)
|
|
158
|
+
wait_seconds = wait_before_retry(attempts)
|
|
159
|
+
logger.warn("delivery_retry", destination: destination_name, delivery_unit:, attempt: attempts,
|
|
160
|
+
wait_seconds:, error_class: e.class.name)
|
|
150
161
|
retry
|
|
151
162
|
end
|
|
152
163
|
end
|
|
153
|
-
# rubocop:enable Metrics/MethodLength
|
|
164
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
154
165
|
|
|
155
166
|
def idempotency_key_for(payload:, delivery_unit:)
|
|
156
167
|
[
|
|
@@ -191,6 +202,7 @@ module Mammoth
|
|
|
191
202
|
def wait_before_retry(attempts)
|
|
192
203
|
wait_seconds = retry_schedule.fetch(attempts - 1, retry_schedule.last)
|
|
193
204
|
sleeper.call(wait_seconds)
|
|
205
|
+
wait_seconds
|
|
194
206
|
end
|
|
195
207
|
|
|
196
208
|
def deliver_to_sink(prepared, work:, delivery_method:)
|
|
@@ -207,9 +219,18 @@ module Mammoth
|
|
|
207
219
|
error: error,
|
|
208
220
|
retry_count: attempts
|
|
209
221
|
)
|
|
222
|
+
logger.error(
|
|
223
|
+
"delivery_dead_lettered",
|
|
224
|
+
destination: destination_name, attempts:, dead_letter_id: id, error_class: error.class.name
|
|
225
|
+
)
|
|
210
226
|
{ status: "dead_lettered", dead_letter_id: id, attempts: attempts }
|
|
211
227
|
end
|
|
212
228
|
|
|
229
|
+
def log_outcome(event, result, delivery_unit:)
|
|
230
|
+
logger.info(event, destination: destination_name, delivery_unit:, status: result[:status],
|
|
231
|
+
attempts: result[:attempts], reason: result[:reason], duplicate: result[:duplicate])
|
|
232
|
+
end
|
|
233
|
+
|
|
213
234
|
def transaction_payload?(payload)
|
|
214
235
|
payload["type"] == TransactionEnvelopeSerializer::PAYLOAD_TYPE
|
|
215
236
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
module Mammoth
|
|
7
|
+
# Container-friendly structured application logging.
|
|
8
|
+
module Logging
|
|
9
|
+
LEVELS = { "debug" => 0, "info" => 1, "warn" => 2, "error" => 3 }.freeze
|
|
10
|
+
|
|
11
|
+
# Logger that emits one JSON object per line.
|
|
12
|
+
class Logger
|
|
13
|
+
attr_reader :level, :output, :clock
|
|
14
|
+
|
|
15
|
+
def initialize(level:, output: $stdout, clock: -> { Time.now.utc })
|
|
16
|
+
@level = level.to_s
|
|
17
|
+
@output = output
|
|
18
|
+
@clock = clock
|
|
19
|
+
raise ConfigurationError, "unsupported logging level: #{@level}" unless LEVELS.key?(@level)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def debug(event, **context) = write("debug", event, context)
|
|
23
|
+
def info(event, **context) = write("info", event, context)
|
|
24
|
+
def warn(event, **context) = write("warn", event, context)
|
|
25
|
+
def error(event, **context) = write("error", event, context)
|
|
26
|
+
|
|
27
|
+
def enabled?(severity)
|
|
28
|
+
LEVELS.fetch(severity.to_s) >= LEVELS.fetch(level)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def write(severity, event, context)
|
|
34
|
+
return false unless enabled?(severity)
|
|
35
|
+
|
|
36
|
+
output.puts JSON.generate(
|
|
37
|
+
{ timestamp: clock.call.utc.iso8601(6), severity:, service: "mammoth", event: }.merge(context.compact)
|
|
38
|
+
)
|
|
39
|
+
true
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# No-op logger used at injectable library boundaries.
|
|
44
|
+
class NullLogger
|
|
45
|
+
INSTANCE = new
|
|
46
|
+
|
|
47
|
+
def debug(_event, **_context) = false
|
|
48
|
+
def info(_event, **_context) = false
|
|
49
|
+
def warn(_event, **_context) = false
|
|
50
|
+
def error(_event, **_context) = false
|
|
51
|
+
def enabled?(_severity) = false
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
module_function
|
|
55
|
+
|
|
56
|
+
def build(config, output: $stdout, clock: -> { Time.now.utc })
|
|
57
|
+
Logger.new(level: config.dig("logging", "level") || "info", output:, clock:)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -11,7 +11,7 @@ module Mammoth
|
|
|
11
11
|
# the same configured operational-state backend.
|
|
12
12
|
class ObservabilityServer
|
|
13
13
|
# Default bind host for the observability server.
|
|
14
|
-
DEFAULT_HOST = "
|
|
14
|
+
DEFAULT_HOST = "127.0.0.1"
|
|
15
15
|
# Default TCP port for the observability server.
|
|
16
16
|
DEFAULT_PORT = 9393
|
|
17
17
|
|
|
@@ -29,7 +29,7 @@ module Mammoth
|
|
|
29
29
|
@port = port || config.dig("observability", "port") || DEFAULT_PORT
|
|
30
30
|
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
31
31
|
@slot_health_provider = slot_health_provider || Sources::Postgres.new(config)
|
|
32
|
-
@logger = logger || WEBrick::Log.new($
|
|
32
|
+
@logger = logger || WEBrick::Log.new($stdout, webrick_log_level)
|
|
33
33
|
@server = build_server
|
|
34
34
|
mount_endpoints
|
|
35
35
|
end
|
|
@@ -50,6 +50,15 @@ module Mammoth
|
|
|
50
50
|
|
|
51
51
|
private
|
|
52
52
|
|
|
53
|
+
def webrick_log_level
|
|
54
|
+
{
|
|
55
|
+
"debug" => WEBrick::Log::DEBUG,
|
|
56
|
+
"info" => WEBrick::Log::INFO,
|
|
57
|
+
"warn" => WEBrick::Log::WARN,
|
|
58
|
+
"error" => WEBrick::Log::ERROR
|
|
59
|
+
}.fetch(config.dig("logging", "level") || "info")
|
|
60
|
+
end
|
|
61
|
+
|
|
53
62
|
def build_server
|
|
54
63
|
WEBrick::HTTPServer.new(
|
|
55
64
|
BindAddress: host,
|
|
@@ -29,6 +29,8 @@ module Mammoth
|
|
|
29
29
|
attr_reader :checkpoint_store
|
|
30
30
|
# @return [#inspect, nil] injected PostgreSQL publication inspector
|
|
31
31
|
attr_reader :publication_inspector
|
|
32
|
+
# @return [Mammoth::Logging::Logger, Mammoth::Logging::NullLogger] operational logger
|
|
33
|
+
attr_reader :logger
|
|
32
34
|
|
|
33
35
|
# Build a PostgreSQL CDC source.
|
|
34
36
|
#
|
|
@@ -40,7 +42,7 @@ module Mammoth
|
|
|
40
42
|
# @param checkpoint_store [Mammoth::CheckpointStore, nil] persisted checkpoints for restart resume
|
|
41
43
|
# @param publication_inspector [#inspect, nil] injected publication metadata inspector
|
|
42
44
|
def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil, checkpoint_store: nil,
|
|
43
|
-
publication_inspector: nil)
|
|
45
|
+
publication_inspector: nil, logger: Logging::NullLogger::INSTANCE)
|
|
44
46
|
@config = config
|
|
45
47
|
@runner = runner
|
|
46
48
|
@parser = parser
|
|
@@ -48,6 +50,7 @@ module Mammoth
|
|
|
48
50
|
@adapter = adapter
|
|
49
51
|
@checkpoint_store = checkpoint_store
|
|
50
52
|
@publication_inspector = publication_inspector
|
|
53
|
+
@logger = logger
|
|
51
54
|
end
|
|
52
55
|
|
|
53
56
|
# Stream CDC::Core work from PostgreSQL logical replication.
|
|
@@ -61,9 +64,12 @@ module Mammoth
|
|
|
61
64
|
# @yieldparam work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
|
|
62
65
|
# @return [Enumerator, nil]
|
|
63
66
|
# @raise [Mammoth::ReplicationError] when the source cannot stream CDC work
|
|
67
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
64
68
|
def each(&block)
|
|
65
69
|
return enum_for(:each) unless block_given?
|
|
66
70
|
|
|
71
|
+
logger.info("replication_starting", slot: config.dig("replication", "slot"),
|
|
72
|
+
publications: Array(config.dig("replication", "publications")))
|
|
67
73
|
preflight_slot!
|
|
68
74
|
preflight_replica_identity!
|
|
69
75
|
normalizer = effective_adapter
|
|
@@ -72,22 +78,29 @@ module Mammoth
|
|
|
72
78
|
end
|
|
73
79
|
|
|
74
80
|
normalizer.each_normalized(decoded_stream) do |work|
|
|
75
|
-
|
|
81
|
+
core_work = validate_core_work!(work)
|
|
82
|
+
logger.debug("replication_work_normalized", work_type: core_work.class.name,
|
|
83
|
+
source_position: @latest_progress_position)
|
|
84
|
+
yield_with_progress_position(core_work, &block)
|
|
76
85
|
end
|
|
86
|
+
logger.info("replication_stopped", slot: config.dig("replication", "slot"))
|
|
77
87
|
nil
|
|
78
88
|
rescue StandardError => e
|
|
89
|
+
logger.error("replication_failed", slot: config.dig("replication", "slot"), error_class: e.class.name)
|
|
79
90
|
raise e if e.is_a?(ReplicationError)
|
|
80
91
|
|
|
81
92
|
raise ReplicationError, "PostgreSQL CDC source failed: #{e.message}"
|
|
82
93
|
end
|
|
94
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
83
95
|
|
|
84
96
|
# Acknowledge a durably handled PostgreSQL WAL position.
|
|
85
97
|
#
|
|
86
98
|
# @param lsn [String, Integer] pgoutput-client compatible WAL position
|
|
87
99
|
# @return [Integer] normalized acknowledged WAL position
|
|
88
100
|
def acknowledge(lsn)
|
|
89
|
-
effective_runner.ack(lsn)
|
|
101
|
+
effective_runner.ack(lsn).tap { logger.debug("wal_acknowledged", slot: config.dig("replication", "slot"), lsn:) }
|
|
90
102
|
rescue StandardError => e
|
|
103
|
+
logger.error("wal_acknowledgement_failed", slot: config.dig("replication", "slot"), error_class: e.class.name)
|
|
91
104
|
raise ReplicationError, "PostgreSQL WAL acknowledgement failed: #{e.message}"
|
|
92
105
|
end
|
|
93
106
|
|
data/lib/mammoth/version.rb
CHANGED
data/lib/mammoth.rb
CHANGED
|
@@ -6,6 +6,7 @@ require_relative "mammoth/version"
|
|
|
6
6
|
require_relative "mammoth/errors"
|
|
7
7
|
require_relative "mammoth/registry"
|
|
8
8
|
require_relative "mammoth/configuration"
|
|
9
|
+
require_relative "mammoth/logging"
|
|
9
10
|
require_relative "mammoth/lifecycle_hooks"
|
|
10
11
|
require_relative "mammoth/node_identity"
|
|
11
12
|
require_relative "mammoth/capabilities"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mammoth
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ken C. Demanawa
|
|
@@ -37,6 +37,26 @@ dependencies:
|
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
39
|
version: '0.1'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: json
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 2.19.2
|
|
47
|
+
- - "<"
|
|
48
|
+
- !ruby/object:Gem::Version
|
|
49
|
+
version: '3.0'
|
|
50
|
+
type: :runtime
|
|
51
|
+
prerelease: false
|
|
52
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
53
|
+
requirements:
|
|
54
|
+
- - ">="
|
|
55
|
+
- !ruby/object:Gem::Version
|
|
56
|
+
version: 2.19.2
|
|
57
|
+
- - "<"
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: '3.0'
|
|
40
60
|
- !ruby/object:Gem::Dependency
|
|
41
61
|
name: json-schema
|
|
42
62
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -211,6 +231,7 @@ files:
|
|
|
211
231
|
- lib/mammoth/event_serializer.rb
|
|
212
232
|
- lib/mammoth/fanout_delivery_worker.rb
|
|
213
233
|
- lib/mammoth/lifecycle_hooks.rb
|
|
234
|
+
- lib/mammoth/logging.rb
|
|
214
235
|
- lib/mammoth/metrics_observer.rb
|
|
215
236
|
- lib/mammoth/node_identity.rb
|
|
216
237
|
- lib/mammoth/observability_metrics.rb
|