mammoth 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +47 -0
- data/README.md +56 -6
- data/config/mammoth.example.yml +7 -2
- data/lib/mammoth/application.rb +32 -5
- data/lib/mammoth/cli.rb +1 -1
- data/lib/mammoth/delivery_progress_coordinator.rb +5 -2
- data/lib/mammoth/delivery_worker.rb +32 -11
- data/lib/mammoth/logging.rb +60 -0
- data/lib/mammoth/observability_server.rb +11 -2
- data/lib/mammoth/sources/postgres.rb +16 -3
- data/lib/mammoth/version.rb +1 -1
- data/lib/mammoth.rb +1 -0
- metadata +22 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7d16f1b05d2ae82569933033416a20892ae7e7afc9cdd282aacd6343789157c3
|
|
4
|
+
data.tar.gz: 1af59d9757dd584f462c5a888246c73693459aff503f6ef775e24aa4c4d19b49
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 14452ab3df0aaae412be115a33b25b20ae3ef9698146e2f0e00c38f0453245ea9abec2a1cdd93a6f780288e722fb982b3fa3f7ab7fafc0de979c19abccd9a74e
|
|
7
|
+
data.tar.gz: 39832167b4069a9b8b8b4bab5ad86f2a3a150449e216d41a4a915fc2e748b471d3361f183b088a7a990402515351d6bf7e3e19c5576a683dabdb463e6e508dce
|
data/CHANGELOG.md
CHANGED
|
@@ -1,7 +1,54 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
# @title Changelog
|
|
3
|
+
-->
|
|
4
|
+
|
|
1
5
|
# Changelog
|
|
2
6
|
|
|
3
7
|
## Unreleased
|
|
4
8
|
|
|
9
|
+
## 1.5.0 - 2026-07-19
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- Default the unauthenticated observability HTTP server to the loopback
|
|
14
|
+
interface and document the network controls required for non-loopback
|
|
15
|
+
bindings.
|
|
16
|
+
- Implement `logging.level` with container-friendly JSON stdout logs for
|
|
17
|
+
application and replication lifecycle, delivery outcomes, retries, dead
|
|
18
|
+
letters, checkpoints, and debug-level work progress.
|
|
19
|
+
- Require a patched `json` gem and remove the vulnerable bundled default-gem
|
|
20
|
+
specification from the runtime image to address CVE-2026-33210.
|
|
21
|
+
|
|
22
|
+
### Quality
|
|
23
|
+
|
|
24
|
+
- Run the container quickstart smoke test on pull requests and whenever Docker,
|
|
25
|
+
application, configuration, or dependency inputs change.
|
|
26
|
+
- Added Dependabot updates, Ruby dependency auditing, dependency review,
|
|
27
|
+
CodeQL analysis, and high- and critical-severity container vulnerability
|
|
28
|
+
scanning.
|
|
29
|
+
- Pin the Ruby container base image to an immutable digest and publish release
|
|
30
|
+
images with SBOM and maximum provenance attestations.
|
|
31
|
+
- Published a full reference benchmark snapshot with machine-readable results,
|
|
32
|
+
raw output, and documented interpretation across projection, delivery,
|
|
33
|
+
fanout, SQLite operational state, observability, and dead-letter replay.
|
|
34
|
+
|
|
35
|
+
## 1.4.0 - 2026-07-19
|
|
36
|
+
|
|
37
|
+
### Added
|
|
38
|
+
|
|
39
|
+
- Declared PostgreSQL 14 through PostgreSQL 18, inclusive, as Mammoth's
|
|
40
|
+
supported server range.
|
|
41
|
+
- Replaced the PostgreSQL 17-only CI gate with real logical-replication E2E
|
|
42
|
+
coverage across PostgreSQL 14, 15, 16, 17, and 18.
|
|
43
|
+
- Documented the maintained-version support policy, version-dependent
|
|
44
|
+
`pg_replication_slots` fields, and PostgreSQL 18-only
|
|
45
|
+
`idle_replication_slot_timeout`.
|
|
46
|
+
|
|
47
|
+
### Fixed
|
|
48
|
+
|
|
49
|
+
- Query `pg_replication_slots.invalidation_reason` in the E2E helper only on
|
|
50
|
+
PostgreSQL 17 and newer; PostgreSQL 16 does not expose that field.
|
|
51
|
+
|
|
5
52
|
## 1.3.0 - 2026-07-19
|
|
6
53
|
|
|
7
54
|
### Added
|
data/README.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
# @markup markdown
|
|
3
|
+
# @title Github Repo README
|
|
4
|
+
# @author Ken C. Demanawa
|
|
5
|
+
-->
|
|
6
|
+
|
|
1
7
|
<p align="center">
|
|
2
8
|
<picture>
|
|
3
9
|
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/kanutocd/mammoth/main/docs/branding/logo/exports/png/mammoth-primary-horizontal-reversed-transparent.png">
|
|
@@ -8,10 +14,13 @@
|
|
|
8
14
|
|
|
9
15
|
# Mammoth
|
|
10
16
|
|
|
11
|
-
[](https://rubygems.org/gems/mammoth)
|
|
18
|
+
[](https://www.ruby-lang.org/)
|
|
19
|
+
[](https://github.com/kanutocd/mammoth/actions/workflows/ci.yml)
|
|
20
|
+
[](https://github.com/kanutocd/mammoth/actions/workflows/security.yml)
|
|
21
|
+
[](https://github.com/kanutocd/mammoth/blob/main/.github/workflows/ci.yml#L50)
|
|
22
|
+
[](LICENSE.txt)
|
|
23
|
+
|
|
15
24
|
|
|
16
25
|
Mammoth is a self-hosted PostgreSQL event relay focused on reliable delivery
|
|
17
26
|
of database change events.
|
|
@@ -32,6 +41,20 @@ Mammoth is intentionally boring infrastructure. It uses YAML configuration,
|
|
|
32
41
|
JSON Schema validation, local SQLite operational state, and the CDC Ecosystem's
|
|
33
42
|
shared vocabulary so operators can inspect, recover, and reason about delivery.
|
|
34
43
|
|
|
44
|
+
## Supported PostgreSQL versions
|
|
45
|
+
|
|
46
|
+
Mammoth supports PostgreSQL 14 through PostgreSQL 18, inclusive. These are the
|
|
47
|
+
PostgreSQL major versions currently maintained by the PostgreSQL community and
|
|
48
|
+
covered by Mammoth's real logical-replication E2E compatibility matrix.
|
|
49
|
+
|
|
50
|
+
Mammoth supports PostgreSQL major versions that are both maintained by the
|
|
51
|
+
PostgreSQL community and included in Mammoth's compatibility test matrix. New
|
|
52
|
+
PostgreSQL majors are unsupported until explicitly tested and documented. EOL
|
|
53
|
+
versions may be removed from the supported range in a subsequent Mammoth minor
|
|
54
|
+
release with release-note notice.
|
|
55
|
+
|
|
56
|
+
PostgreSQL 19 is a development release and is not supported.
|
|
57
|
+
|
|
35
58
|
## Documentation
|
|
36
59
|
|
|
37
60
|
Documentation site:
|
|
@@ -84,6 +107,7 @@ Mammoth 1.x includes:
|
|
|
84
107
|
- per-destination payload removal and masking policies
|
|
85
108
|
- per-destination enable/disable and retry policy controls
|
|
86
109
|
- delivery worker with retry, delivered-ledger, and DLQ handling
|
|
110
|
+
- configurable `debug`, `info`, `warn`, and `error` JSON logging to stdout
|
|
87
111
|
- contiguous delivery watermark for checkpoint and PostgreSQL acknowledgement
|
|
88
112
|
- source-owned transport LSN preservation independent of payload `commit_lsn`
|
|
89
113
|
- fail-closed PostgreSQL slot and checkpoint continuity preflight
|
|
@@ -235,6 +259,30 @@ bundle exec ./exe/mammoth start config/mammoth.example.yml
|
|
|
235
259
|
bundle exec ./exe/mammoth observability config/mammoth.example.yml
|
|
236
260
|
```
|
|
237
261
|
|
|
262
|
+
## Logging
|
|
263
|
+
|
|
264
|
+
Mammoth emits one JSON object per line to standard output. Set
|
|
265
|
+
`logging.level` to `debug`, `info`, `warn`, or `error`; `info` is the
|
|
266
|
+
recommended normal setting, while `debug` adds per-work and WAL acknowledgement
|
|
267
|
+
records.
|
|
268
|
+
|
|
269
|
+
```yaml
|
|
270
|
+
logging:
|
|
271
|
+
level: info
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
In Docker Compose, follow the application, replication, delivery, retry,
|
|
275
|
+
dead-letter, and checkpoint activity with:
|
|
276
|
+
|
|
277
|
+
```bash
|
|
278
|
+
docker compose logs -f mammoth
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Operational identifiers are included, but payload bodies, configured headers,
|
|
282
|
+
credentials, signing secrets, and exception messages are excluded. See
|
|
283
|
+
[`docs/CONFIGURATION.md`](docs/CONFIGURATION.md#logging) for the event and level
|
|
284
|
+
details.
|
|
285
|
+
|
|
238
286
|
Reconstruct and deliver a single persisted event JSON file through Mammoth's
|
|
239
287
|
core delivery path:
|
|
240
288
|
|
|
@@ -270,8 +318,10 @@ production:
|
|
|
270
318
|
- observability snapshots
|
|
271
319
|
- DLQ replay
|
|
272
320
|
|
|
273
|
-
The
|
|
274
|
-
|
|
321
|
+
The [reference snapshot](https://github.com/kanutocd/mammoth/blob/main/benchmark/results/20260724T104950Z/snapshot.md)
|
|
322
|
+
records a full local run, its exact commands, environment, raw output, and
|
|
323
|
+
interpretation. It is a reproducible baseline, not a universal performance
|
|
324
|
+
claim. Re-run the scripts in
|
|
275
325
|
[`benchmark/`](https://github.com/kanutocd/mammoth/tree/main/benchmark) on your own hardware when choosing
|
|
276
326
|
`runtime.concurrency`, `destinations`, SQLite storage, scrape frequency, and
|
|
277
327
|
DLQ replay expectations.
|
data/config/mammoth.example.yml
CHANGED
|
@@ -385,7 +385,11 @@ operational_state:
|
|
|
385
385
|
|
|
386
386
|
observability:
|
|
387
387
|
# Bind host for the optional health, readiness, and metrics server.
|
|
388
|
-
|
|
388
|
+
#
|
|
389
|
+
# The endpoints do not authenticate clients and may expose operational
|
|
390
|
+
# details. Keep the loopback default unless a trusted network boundary or
|
|
391
|
+
# authenticated reverse proxy protects the listener.
|
|
392
|
+
host: 127.0.0.1
|
|
389
393
|
|
|
390
394
|
# HTTP port for /healthz, /readyz, and /metrics.
|
|
391
395
|
port: 9393
|
|
@@ -397,7 +401,8 @@ logging:
|
|
|
397
401
|
# Very chatty; useful while diagnosing delivery or replication behavior.
|
|
398
402
|
#
|
|
399
403
|
# info:
|
|
400
|
-
# Recommended normal default.
|
|
404
|
+
# Recommended normal default. Emits lifecycle, delivery, checkpoint, retry,
|
|
405
|
+
# and dead-letter JSON records to stdout for container log collection.
|
|
401
406
|
#
|
|
402
407
|
# warn/error:
|
|
403
408
|
# Quieter production modes.
|
data/lib/mammoth/application.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Mammoth
|
|
|
12
12
|
# rubocop:disable Metrics/ClassLength
|
|
13
13
|
class Application
|
|
14
14
|
attr_reader :config, :state_adapter, :consumer, :delivery_worker, :checkpoint_store, :lifecycle_hooks, :observer,
|
|
15
|
-
:progress_coordinator
|
|
15
|
+
:progress_coordinator, :logger
|
|
16
16
|
|
|
17
17
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
18
18
|
# @param source [#each, nil] injectable event source for tests and demos
|
|
@@ -21,9 +21,11 @@ module Mammoth
|
|
|
21
21
|
# @param sleeper [#call] retry sleep strategy
|
|
22
22
|
# @param lifecycle_hooks [Mammoth::LifecycleHooks, Hash] local lifecycle callbacks
|
|
23
23
|
# @param observer [CDC::Core::Observer, nil] dispatch lifecycle observer
|
|
24
|
+
# @param logger [Mammoth::Logging::Logger, nil] structured operational logger
|
|
24
25
|
def initialize(config, source: nil, sink: nil, state_adapter: nil, sleeper: Kernel.method(:sleep),
|
|
25
|
-
lifecycle_hooks: LifecycleHooks.new, observer: nil)
|
|
26
|
+
lifecycle_hooks: LifecycleHooks.new, observer: nil, logger: nil)
|
|
26
27
|
@config = config
|
|
28
|
+
@logger = logger || Logging.build(config)
|
|
27
29
|
@lifecycle_hooks = build_lifecycle_hooks(lifecycle_hooks)
|
|
28
30
|
@observer = observer || MetricsObserver.new
|
|
29
31
|
@state_adapter = state_adapter || build_state_adapter
|
|
@@ -49,22 +51,37 @@ module Mammoth
|
|
|
49
51
|
# Start the application runtime and deliver consumed CDC work.
|
|
50
52
|
#
|
|
51
53
|
# @return [Integer] number of processed work units
|
|
54
|
+
# rubocop:disable Metrics/AbcSize
|
|
52
55
|
def start
|
|
53
56
|
runtime = build_runtime
|
|
54
57
|
processed = nil
|
|
55
58
|
|
|
59
|
+
operational_logger.info("application_started", mammoth_name: config&.dig("mammoth", "name"),
|
|
60
|
+
runtime: config&.dig("runtime", "adapter"),
|
|
61
|
+
delivery_unit: config&.dig("delivery", "unit"))
|
|
56
62
|
lifecycle_hooks.call(:before_start, application_context(runtime: runtime))
|
|
57
63
|
processed = process_consumer(runtime)
|
|
58
64
|
lifecycle_hooks.call(:after_start, application_context(runtime: runtime, processed: processed))
|
|
65
|
+
operational_logger.info("application_completed", mammoth_name: config&.dig("mammoth", "name"), processed:)
|
|
59
66
|
processed
|
|
67
|
+
rescue StandardError => e
|
|
68
|
+
operational_logger.error("application_failed", mammoth_name: config&.dig("mammoth", "name"),
|
|
69
|
+
error_class: e.class.name)
|
|
70
|
+
raise
|
|
60
71
|
ensure
|
|
61
72
|
lifecycle_hooks.call(:before_shutdown, application_context(runtime: runtime, processed: processed))
|
|
62
73
|
runtime.shutdown if runtime.respond_to?(:shutdown)
|
|
63
74
|
lifecycle_hooks.call(:after_shutdown, application_context(runtime: runtime, processed: processed))
|
|
75
|
+
operational_logger.info("application_stopped", mammoth_name: config&.dig("mammoth", "name"), processed:)
|
|
64
76
|
end
|
|
77
|
+
# rubocop:enable Metrics/AbcSize
|
|
65
78
|
|
|
66
79
|
private
|
|
67
80
|
|
|
81
|
+
def operational_logger
|
|
82
|
+
logger || Logging::NullLogger::INSTANCE
|
|
83
|
+
end
|
|
84
|
+
|
|
68
85
|
def build_lifecycle_hooks(hooks)
|
|
69
86
|
return hooks if hooks.is_a?(LifecycleHooks)
|
|
70
87
|
|
|
@@ -84,6 +101,7 @@ module Mammoth
|
|
|
84
101
|
processed = 0
|
|
85
102
|
|
|
86
103
|
consumer.start_with_boundaries do |work, group_end|
|
|
104
|
+
operational_logger.debug("work_received", work_type: work.class.name, work_id: work_identifier(work), group_end:)
|
|
87
105
|
progress_coordinator.register(work, group_end:)
|
|
88
106
|
process_work(runtime, work)
|
|
89
107
|
processed += 1
|
|
@@ -94,6 +112,13 @@ module Mammoth
|
|
|
94
112
|
processed
|
|
95
113
|
end
|
|
96
114
|
|
|
115
|
+
def work_identifier(work)
|
|
116
|
+
return work.transaction_id if work.respond_to?(:transaction_id)
|
|
117
|
+
return work.event_id if work.respond_to?(:event_id)
|
|
118
|
+
|
|
119
|
+
nil
|
|
120
|
+
end
|
|
121
|
+
|
|
97
122
|
def process_work(runtime, work)
|
|
98
123
|
runtime.process(work)
|
|
99
124
|
end
|
|
@@ -115,7 +140,7 @@ module Mammoth
|
|
|
115
140
|
end
|
|
116
141
|
|
|
117
142
|
def build_source
|
|
118
|
-
Sources::Postgres.new(config, checkpoint_store: checkpoint_store)
|
|
143
|
+
Sources::Postgres.new(config, checkpoint_store: checkpoint_store, logger: logger)
|
|
119
144
|
end
|
|
120
145
|
|
|
121
146
|
def build_progress_coordinator
|
|
@@ -125,7 +150,8 @@ module Mammoth
|
|
|
125
150
|
slot_name: config.dig("replication", "slot"),
|
|
126
151
|
publication_name: Array(config.dig("replication", "publications")).join(","),
|
|
127
152
|
acknowledger: source_acknowledger,
|
|
128
|
-
position_resolver: source_position_resolver
|
|
153
|
+
position_resolver: source_position_resolver,
|
|
154
|
+
logger: logger
|
|
129
155
|
)
|
|
130
156
|
end
|
|
131
157
|
|
|
@@ -147,7 +173,8 @@ module Mammoth
|
|
|
147
173
|
dead_letter_store: state_adapter.dead_letter_store,
|
|
148
174
|
delivered_envelope_store: state_adapter.delivered_envelope_store,
|
|
149
175
|
sleeper: sleeper,
|
|
150
|
-
delivery_policy: delivery_policy
|
|
176
|
+
delivery_policy: delivery_policy,
|
|
177
|
+
logger: logger
|
|
151
178
|
)
|
|
152
179
|
end
|
|
153
180
|
|
data/lib/mammoth/cli.rb
CHANGED
|
@@ -109,7 +109,7 @@ module Mammoth
|
|
|
109
109
|
def observability
|
|
110
110
|
config = load_config
|
|
111
111
|
server = ObservabilityServer.new(config)
|
|
112
|
-
|
|
112
|
+
Logging.build(config).info("observability_started", host: server.host, port: server.port)
|
|
113
113
|
server.start
|
|
114
114
|
0
|
|
115
115
|
end
|
|
@@ -36,7 +36,7 @@ module Mammoth
|
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
attr_reader :checkpoint_store, :source_name, :slot_name, :publication_name
|
|
39
|
+
attr_reader :checkpoint_store, :source_name, :slot_name, :publication_name, :logger
|
|
40
40
|
|
|
41
41
|
# @param checkpoint_store [Mammoth::CheckpointStore] durable checkpoint persistence
|
|
42
42
|
# @param source_name [String] logical source name
|
|
@@ -45,13 +45,14 @@ module Mammoth
|
|
|
45
45
|
# @param acknowledger [#call, nil] upstream durable-progress acknowledgement
|
|
46
46
|
# @param position_resolver [#call, nil] source-owned durable position resolver
|
|
47
47
|
def initialize(checkpoint_store:, source_name:, slot_name:, publication_name:, acknowledger: nil,
|
|
48
|
-
position_resolver: nil)
|
|
48
|
+
position_resolver: nil, logger: Logging::NullLogger::INSTANCE)
|
|
49
49
|
@checkpoint_store = checkpoint_store
|
|
50
50
|
@source_name = source_name
|
|
51
51
|
@slot_name = slot_name
|
|
52
52
|
@publication_name = publication_name
|
|
53
53
|
@acknowledger = acknowledger
|
|
54
54
|
@position_resolver = position_resolver
|
|
55
|
+
@logger = logger
|
|
55
56
|
# rubocop:disable Layout/LeadingCommentSpace -- Steep inline type syntax requires `#:`.
|
|
56
57
|
@groups = [] #: Array[Group]
|
|
57
58
|
@entries_by_work = {} #: Hash[untyped, Array[Entry]]
|
|
@@ -133,6 +134,8 @@ module Mammoth
|
|
|
133
134
|
last_lsn: position
|
|
134
135
|
)
|
|
135
136
|
@acknowledger&.call(position)
|
|
137
|
+
logger.info("checkpoint_advanced", mammoth_name: source_name, slot: slot_name, publication: publication_name,
|
|
138
|
+
source_position: position)
|
|
136
139
|
end
|
|
137
140
|
|
|
138
141
|
def remove_group(group)
|
|
@@ -7,12 +7,12 @@ module Mammoth
|
|
|
7
7
|
# the delivery contract small: attempt webhook delivery, record idempotent
|
|
8
8
|
# success, and persist exhausted failures to the dead letter queue. Contiguous
|
|
9
9
|
# checkpointing is owned by DeliveryProgressCoordinator.
|
|
10
|
-
class DeliveryWorker
|
|
10
|
+
class DeliveryWorker # rubocop:disable Metrics/ClassLength
|
|
11
11
|
# Default source name used when an event does not provide one.
|
|
12
12
|
DEFAULT_SOURCE = "postgresql"
|
|
13
13
|
|
|
14
14
|
attr_reader :sink, :checkpoint_store, :dead_letter_store, :delivered_envelope_store, :retry_schedule, :max_attempts,
|
|
15
|
-
:sleeper, :source_name, :slot_name, :publication_name, :route_filter, :payload_policy, :enabled
|
|
15
|
+
:sleeper, :source_name, :slot_name, :publication_name, :route_filter, :payload_policy, :enabled, :logger
|
|
16
16
|
|
|
17
17
|
# @param sink [#deliver] destination sink
|
|
18
18
|
# @param checkpoint_store [Mammoth::CheckpointStore] retained dependency; shared progress owns writes
|
|
@@ -29,7 +29,7 @@ module Mammoth
|
|
|
29
29
|
# @param enabled [Boolean] whether this destination accepts new deliveries
|
|
30
30
|
def initialize(sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:, source_name:, slot_name:,
|
|
31
31
|
publication_name:, max_attempts:, retry_schedule:, sleeper: Kernel.method(:sleep),
|
|
32
|
-
route_filter: nil, payload_policy: nil, enabled: true)
|
|
32
|
+
route_filter: nil, payload_policy: nil, enabled: true, logger: Logging::NullLogger::INSTANCE)
|
|
33
33
|
@sink = sink
|
|
34
34
|
@checkpoint_store = checkpoint_store
|
|
35
35
|
@dead_letter_store = dead_letter_store
|
|
@@ -43,6 +43,7 @@ module Mammoth
|
|
|
43
43
|
@route_filter = route_filter || RouteFilter.new
|
|
44
44
|
@payload_policy = payload_policy || PayloadPolicy.new
|
|
45
45
|
@enabled = enabled
|
|
46
|
+
@logger = logger
|
|
46
47
|
return unless @payload_policy.active? && !sink.respond_to?(:deliver_payload)
|
|
47
48
|
|
|
48
49
|
raise ConfigurationError, "destination #{destination_name} does not accept prepared payloads"
|
|
@@ -58,7 +59,7 @@ module Mammoth
|
|
|
58
59
|
# @param sleeper [#call] sleep strategy
|
|
59
60
|
# @return [Mammoth::DeliveryWorker]
|
|
60
61
|
def self.from_config(config, sink:, checkpoint_store:, dead_letter_store:, delivered_envelope_store:,
|
|
61
|
-
sleeper: Kernel.method(:sleep), delivery_policy: {})
|
|
62
|
+
sleeper: Kernel.method(:sleep), delivery_policy: {}, logger: Logging::NullLogger::INSTANCE)
|
|
62
63
|
new(
|
|
63
64
|
sink: sink,
|
|
64
65
|
checkpoint_store: checkpoint_store,
|
|
@@ -72,7 +73,8 @@ module Mammoth
|
|
|
72
73
|
sleeper: sleeper,
|
|
73
74
|
route_filter: delivery_policy.fetch("route_filter", RouteFilter.new),
|
|
74
75
|
payload_policy: delivery_policy.fetch("payload_policy", PayloadPolicy.new),
|
|
75
|
-
enabled: delivery_policy.fetch("enabled", true)
|
|
76
|
+
enabled: delivery_policy.fetch("enabled", true),
|
|
77
|
+
logger: logger
|
|
76
78
|
)
|
|
77
79
|
end
|
|
78
80
|
|
|
@@ -109,7 +111,7 @@ module Mammoth
|
|
|
109
111
|
deliver_prepared(prepared, work: work, delivery_method: delivery_method)
|
|
110
112
|
end
|
|
111
113
|
|
|
112
|
-
# rubocop:disable Metrics/MethodLength
|
|
114
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
113
115
|
def deliver_prepared(prepared, work:, delivery_method:)
|
|
114
116
|
attempts = 0
|
|
115
117
|
canonical_payload = prepared.canonical_payload
|
|
@@ -118,16 +120,21 @@ module Mammoth
|
|
|
118
120
|
idempotency_key = idempotency_key_for(payload: canonical_payload, delivery_unit: delivery_unit)
|
|
119
121
|
|
|
120
122
|
skip_result = skip_result_for(canonical_payload, idempotency_key: idempotency_key)
|
|
121
|
-
|
|
123
|
+
if skip_result
|
|
124
|
+
log_outcome("delivery_skipped", skip_result, delivery_unit:)
|
|
125
|
+
return skip_result
|
|
126
|
+
end
|
|
122
127
|
|
|
123
128
|
if delivered_envelope_store.delivered?(idempotency_key)
|
|
124
|
-
|
|
129
|
+
result = {
|
|
125
130
|
status: "skipped",
|
|
126
131
|
duplicate: true,
|
|
127
132
|
idempotency_key: idempotency_key,
|
|
128
133
|
attempts: attempts,
|
|
129
134
|
destination: destination_name
|
|
130
135
|
}
|
|
136
|
+
log_outcome("delivery_skipped", result, delivery_unit:)
|
|
137
|
+
return result
|
|
131
138
|
end
|
|
132
139
|
|
|
133
140
|
begin
|
|
@@ -142,15 +149,19 @@ module Mammoth
|
|
|
142
149
|
transaction_id: payload["transaction_id"],
|
|
143
150
|
source_position: payload["source_position"]
|
|
144
151
|
)
|
|
145
|
-
result.merge(attempts: attempts, idempotency_key: idempotency_key)
|
|
152
|
+
result = result.merge(attempts: attempts, idempotency_key: idempotency_key)
|
|
153
|
+
log_outcome("delivery_succeeded", result, delivery_unit:)
|
|
154
|
+
result
|
|
146
155
|
rescue DeliveryError => e
|
|
147
156
|
return dead_letter(payload, e, attempts) if attempts >= max_attempts
|
|
148
157
|
|
|
149
|
-
wait_before_retry(attempts)
|
|
158
|
+
wait_seconds = wait_before_retry(attempts)
|
|
159
|
+
logger.warn("delivery_retry", destination: destination_name, delivery_unit:, attempt: attempts,
|
|
160
|
+
wait_seconds:, error_class: e.class.name)
|
|
150
161
|
retry
|
|
151
162
|
end
|
|
152
163
|
end
|
|
153
|
-
# rubocop:enable Metrics/MethodLength
|
|
164
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
154
165
|
|
|
155
166
|
def idempotency_key_for(payload:, delivery_unit:)
|
|
156
167
|
[
|
|
@@ -191,6 +202,7 @@ module Mammoth
|
|
|
191
202
|
def wait_before_retry(attempts)
|
|
192
203
|
wait_seconds = retry_schedule.fetch(attempts - 1, retry_schedule.last)
|
|
193
204
|
sleeper.call(wait_seconds)
|
|
205
|
+
wait_seconds
|
|
194
206
|
end
|
|
195
207
|
|
|
196
208
|
def deliver_to_sink(prepared, work:, delivery_method:)
|
|
@@ -207,9 +219,18 @@ module Mammoth
|
|
|
207
219
|
error: error,
|
|
208
220
|
retry_count: attempts
|
|
209
221
|
)
|
|
222
|
+
logger.error(
|
|
223
|
+
"delivery_dead_lettered",
|
|
224
|
+
destination: destination_name, attempts:, dead_letter_id: id, error_class: error.class.name
|
|
225
|
+
)
|
|
210
226
|
{ status: "dead_lettered", dead_letter_id: id, attempts: attempts }
|
|
211
227
|
end
|
|
212
228
|
|
|
229
|
+
def log_outcome(event, result, delivery_unit:)
|
|
230
|
+
logger.info(event, destination: destination_name, delivery_unit:, status: result[:status],
|
|
231
|
+
attempts: result[:attempts], reason: result[:reason], duplicate: result[:duplicate])
|
|
232
|
+
end
|
|
233
|
+
|
|
213
234
|
def transaction_payload?(payload)
|
|
214
235
|
payload["type"] == TransactionEnvelopeSerializer::PAYLOAD_TYPE
|
|
215
236
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
module Mammoth
|
|
7
|
+
# Container-friendly structured application logging.
|
|
8
|
+
module Logging
|
|
9
|
+
LEVELS = { "debug" => 0, "info" => 1, "warn" => 2, "error" => 3 }.freeze
|
|
10
|
+
|
|
11
|
+
# Logger that emits one JSON object per line.
|
|
12
|
+
class Logger
|
|
13
|
+
attr_reader :level, :output, :clock
|
|
14
|
+
|
|
15
|
+
def initialize(level:, output: $stdout, clock: -> { Time.now.utc })
|
|
16
|
+
@level = level.to_s
|
|
17
|
+
@output = output
|
|
18
|
+
@clock = clock
|
|
19
|
+
raise ConfigurationError, "unsupported logging level: #{@level}" unless LEVELS.key?(@level)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def debug(event, **context) = write("debug", event, context)
|
|
23
|
+
def info(event, **context) = write("info", event, context)
|
|
24
|
+
def warn(event, **context) = write("warn", event, context)
|
|
25
|
+
def error(event, **context) = write("error", event, context)
|
|
26
|
+
|
|
27
|
+
def enabled?(severity)
|
|
28
|
+
LEVELS.fetch(severity.to_s) >= LEVELS.fetch(level)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def write(severity, event, context)
|
|
34
|
+
return false unless enabled?(severity)
|
|
35
|
+
|
|
36
|
+
output.puts JSON.generate(
|
|
37
|
+
{ timestamp: clock.call.utc.iso8601(6), severity:, service: "mammoth", event: }.merge(context.compact)
|
|
38
|
+
)
|
|
39
|
+
true
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# No-op logger used at injectable library boundaries.
|
|
44
|
+
class NullLogger
|
|
45
|
+
INSTANCE = new
|
|
46
|
+
|
|
47
|
+
def debug(_event, **_context) = false
|
|
48
|
+
def info(_event, **_context) = false
|
|
49
|
+
def warn(_event, **_context) = false
|
|
50
|
+
def error(_event, **_context) = false
|
|
51
|
+
def enabled?(_severity) = false
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
module_function
|
|
55
|
+
|
|
56
|
+
def build(config, output: $stdout, clock: -> { Time.now.utc })
|
|
57
|
+
Logger.new(level: config.dig("logging", "level") || "info", output:, clock:)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -11,7 +11,7 @@ module Mammoth
|
|
|
11
11
|
# the same configured operational-state backend.
|
|
12
12
|
class ObservabilityServer
|
|
13
13
|
# Default bind host for the observability server.
|
|
14
|
-
DEFAULT_HOST = "
|
|
14
|
+
DEFAULT_HOST = "127.0.0.1"
|
|
15
15
|
# Default TCP port for the observability server.
|
|
16
16
|
DEFAULT_PORT = 9393
|
|
17
17
|
|
|
@@ -29,7 +29,7 @@ module Mammoth
|
|
|
29
29
|
@port = port || config.dig("observability", "port") || DEFAULT_PORT
|
|
30
30
|
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
31
31
|
@slot_health_provider = slot_health_provider || Sources::Postgres.new(config)
|
|
32
|
-
@logger = logger || WEBrick::Log.new($
|
|
32
|
+
@logger = logger || WEBrick::Log.new($stdout, webrick_log_level)
|
|
33
33
|
@server = build_server
|
|
34
34
|
mount_endpoints
|
|
35
35
|
end
|
|
@@ -50,6 +50,15 @@ module Mammoth
|
|
|
50
50
|
|
|
51
51
|
private
|
|
52
52
|
|
|
53
|
+
def webrick_log_level
|
|
54
|
+
{
|
|
55
|
+
"debug" => WEBrick::Log::DEBUG,
|
|
56
|
+
"info" => WEBrick::Log::INFO,
|
|
57
|
+
"warn" => WEBrick::Log::WARN,
|
|
58
|
+
"error" => WEBrick::Log::ERROR
|
|
59
|
+
}.fetch(config.dig("logging", "level") || "info")
|
|
60
|
+
end
|
|
61
|
+
|
|
53
62
|
def build_server
|
|
54
63
|
WEBrick::HTTPServer.new(
|
|
55
64
|
BindAddress: host,
|
|
@@ -29,6 +29,8 @@ module Mammoth
|
|
|
29
29
|
attr_reader :checkpoint_store
|
|
30
30
|
# @return [#inspect, nil] injected PostgreSQL publication inspector
|
|
31
31
|
attr_reader :publication_inspector
|
|
32
|
+
# @return [Mammoth::Logging::Logger, Mammoth::Logging::NullLogger] operational logger
|
|
33
|
+
attr_reader :logger
|
|
32
34
|
|
|
33
35
|
# Build a PostgreSQL CDC source.
|
|
34
36
|
#
|
|
@@ -40,7 +42,7 @@ module Mammoth
|
|
|
40
42
|
# @param checkpoint_store [Mammoth::CheckpointStore, nil] persisted checkpoints for restart resume
|
|
41
43
|
# @param publication_inspector [#inspect, nil] injected publication metadata inspector
|
|
42
44
|
def initialize(config, runner: nil, parser: nil, decoder: nil, adapter: nil, checkpoint_store: nil,
|
|
43
|
-
publication_inspector: nil)
|
|
45
|
+
publication_inspector: nil, logger: Logging::NullLogger::INSTANCE)
|
|
44
46
|
@config = config
|
|
45
47
|
@runner = runner
|
|
46
48
|
@parser = parser
|
|
@@ -48,6 +50,7 @@ module Mammoth
|
|
|
48
50
|
@adapter = adapter
|
|
49
51
|
@checkpoint_store = checkpoint_store
|
|
50
52
|
@publication_inspector = publication_inspector
|
|
53
|
+
@logger = logger
|
|
51
54
|
end
|
|
52
55
|
|
|
53
56
|
# Stream CDC::Core work from PostgreSQL logical replication.
|
|
@@ -61,9 +64,12 @@ module Mammoth
|
|
|
61
64
|
# @yieldparam work [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope]
|
|
62
65
|
# @return [Enumerator, nil]
|
|
63
66
|
# @raise [Mammoth::ReplicationError] when the source cannot stream CDC work
|
|
67
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
64
68
|
def each(&block)
|
|
65
69
|
return enum_for(:each) unless block_given?
|
|
66
70
|
|
|
71
|
+
logger.info("replication_starting", slot: config.dig("replication", "slot"),
|
|
72
|
+
publications: Array(config.dig("replication", "publications")))
|
|
67
73
|
preflight_slot!
|
|
68
74
|
preflight_replica_identity!
|
|
69
75
|
normalizer = effective_adapter
|
|
@@ -72,22 +78,29 @@ module Mammoth
|
|
|
72
78
|
end
|
|
73
79
|
|
|
74
80
|
normalizer.each_normalized(decoded_stream) do |work|
|
|
75
|
-
|
|
81
|
+
core_work = validate_core_work!(work)
|
|
82
|
+
logger.debug("replication_work_normalized", work_type: core_work.class.name,
|
|
83
|
+
source_position: @latest_progress_position)
|
|
84
|
+
yield_with_progress_position(core_work, &block)
|
|
76
85
|
end
|
|
86
|
+
logger.info("replication_stopped", slot: config.dig("replication", "slot"))
|
|
77
87
|
nil
|
|
78
88
|
rescue StandardError => e
|
|
89
|
+
logger.error("replication_failed", slot: config.dig("replication", "slot"), error_class: e.class.name)
|
|
79
90
|
raise e if e.is_a?(ReplicationError)
|
|
80
91
|
|
|
81
92
|
raise ReplicationError, "PostgreSQL CDC source failed: #{e.message}"
|
|
82
93
|
end
|
|
94
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
83
95
|
|
|
84
96
|
# Acknowledge a durably handled PostgreSQL WAL position.
|
|
85
97
|
#
|
|
86
98
|
# @param lsn [String, Integer] pgoutput-client compatible WAL position
|
|
87
99
|
# @return [Integer] normalized acknowledged WAL position
|
|
88
100
|
def acknowledge(lsn)
|
|
89
|
-
effective_runner.ack(lsn)
|
|
101
|
+
effective_runner.ack(lsn).tap { logger.debug("wal_acknowledged", slot: config.dig("replication", "slot"), lsn:) }
|
|
90
102
|
rescue StandardError => e
|
|
103
|
+
logger.error("wal_acknowledgement_failed", slot: config.dig("replication", "slot"), error_class: e.class.name)
|
|
91
104
|
raise ReplicationError, "PostgreSQL WAL acknowledgement failed: #{e.message}"
|
|
92
105
|
end
|
|
93
106
|
|
data/lib/mammoth/version.rb
CHANGED
data/lib/mammoth.rb
CHANGED
|
@@ -6,6 +6,7 @@ require_relative "mammoth/version"
|
|
|
6
6
|
require_relative "mammoth/errors"
|
|
7
7
|
require_relative "mammoth/registry"
|
|
8
8
|
require_relative "mammoth/configuration"
|
|
9
|
+
require_relative "mammoth/logging"
|
|
9
10
|
require_relative "mammoth/lifecycle_hooks"
|
|
10
11
|
require_relative "mammoth/node_identity"
|
|
11
12
|
require_relative "mammoth/capabilities"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mammoth
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ken C. Demanawa
|
|
@@ -37,6 +37,26 @@ dependencies:
|
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
39
|
version: '0.1'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: json
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 2.19.2
|
|
47
|
+
- - "<"
|
|
48
|
+
- !ruby/object:Gem::Version
|
|
49
|
+
version: '3.0'
|
|
50
|
+
type: :runtime
|
|
51
|
+
prerelease: false
|
|
52
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
53
|
+
requirements:
|
|
54
|
+
- - ">="
|
|
55
|
+
- !ruby/object:Gem::Version
|
|
56
|
+
version: 2.19.2
|
|
57
|
+
- - "<"
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: '3.0'
|
|
40
60
|
- !ruby/object:Gem::Dependency
|
|
41
61
|
name: json-schema
|
|
42
62
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -211,6 +231,7 @@ files:
|
|
|
211
231
|
- lib/mammoth/event_serializer.rb
|
|
212
232
|
- lib/mammoth/fanout_delivery_worker.rb
|
|
213
233
|
- lib/mammoth/lifecycle_hooks.rb
|
|
234
|
+
- lib/mammoth/logging.rb
|
|
214
235
|
- lib/mammoth/metrics_observer.rb
|
|
215
236
|
- lib/mammoth/node_identity.rb
|
|
216
237
|
- lib/mammoth/observability_metrics.rb
|