mammoth 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +25 -4
- data/config/mammoth.schema.json +1 -1
- data/lib/mammoth/application.rb +20 -40
- data/lib/mammoth/cli.rb +1 -13
- data/lib/mammoth/commands/bootstrap_command.rb +11 -7
- data/lib/mammoth/commands/dead_letters_command.rb +5 -3
- data/lib/mammoth/commands/deliver_sample_command.rb +1 -1
- data/lib/mammoth/commands/status_command.rb +5 -5
- data/lib/mammoth/concurrent_delivery_runtime.rb +15 -3
- data/lib/mammoth/dead_letter_commands.rb +16 -25
- data/lib/mammoth/dead_letter_store.rb +1 -1
- data/lib/mammoth/delivery_processor.rb +46 -7
- data/lib/mammoth/delivery_worker.rb +10 -8
- data/lib/mammoth/dispatch_metrics.rb +55 -0
- data/lib/mammoth/event_serializer.rb +23 -12
- data/lib/mammoth/fanout_delivery_worker.rb +4 -4
- data/lib/mammoth/metrics_observer.rb +52 -0
- data/lib/mammoth/observability_metrics.rb +69 -0
- data/lib/mammoth/observability_server.rb +6 -6
- data/lib/mammoth/observability_snapshot.rb +59 -60
- data/lib/mammoth/operational_state/adapter.rb +16 -0
- data/lib/mammoth/operational_state/registry.rb +8 -0
- data/lib/mammoth/operational_state/sqlite_adapter.rb +18 -2
- data/lib/mammoth/persisted_payload_deserializer.rb +85 -0
- data/lib/mammoth/replication_consumer.rb +15 -47
- data/lib/mammoth/runtimes/batching_runtime.rb +59 -0
- data/lib/mammoth/runtimes/concurrent_adapter.rb +4 -2
- data/lib/mammoth/runtimes/inline_adapter.rb +19 -5
- data/lib/mammoth/runtimes/registry.rb +3 -2
- data/lib/mammoth/sources/postgres.rb +10 -1
- data/lib/mammoth/status.rb +22 -18
- data/lib/mammoth/transaction_envelope_serializer.rb +13 -15
- data/lib/mammoth/version.rb +1 -1
- data/lib/mammoth/webhook_sink.rb +2 -2
- data/lib/mammoth.rb +8 -1
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6333539be1c721f39698edbb27f8285e5b26755b3838dc3f342a198f93b4ff56
|
|
4
|
+
data.tar.gz: 1c825fea3c20668a1095e69a1931abd01f95bd8eb247365ce1b702e99bb40360
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d2470011cc7d16c9d76afde146d529dc7c56963ade4819e77fa75a81c43b7fc535ba57d7e6291eeea94547365a096e38db9182f0c7af01cc011f05cff81ef5a6
|
|
7
|
+
data.tar.gz: b06a8a8d35914d9d123fc7843e3c1318624ec5250c3b89656537680ae0bba9b22dfdf8b4bcb278769e3d49d4fabea4571d107119b19fe824b274e8345e45e0e4
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,44 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.9.0
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- Refactored observability, status, bootstrap, and dead-letter command
|
|
10
|
+
composition to consume `OperationalState::Adapter` instead of opening SQLite
|
|
11
|
+
or constructing concrete stores.
|
|
12
|
+
- Added adapter-owned `bootstrap!`, `ready?`, and configured registry
|
|
13
|
+
composition for backend-neutral operational tooling.
|
|
14
|
+
- Inject the configured operational-state delivered-envelope store into every
|
|
15
|
+
delivery worker instead of deriving a SQLite ledger from the checkpoint store.
|
|
16
|
+
- Made `Mammoth::DeliveryProcessor` implement `CDC::Core::Processor` and return
|
|
17
|
+
`CDC::Core::ProcessorResult` for successful, skipped, and failed deliveries.
|
|
18
|
+
- Wired `CDC::Core::Observer` notifications through inline and concurrent
|
|
19
|
+
runtimes and translated canonical dispatch metrics into Mammoth Prometheus
|
|
20
|
+
counters.
|
|
21
|
+
- Replaced duck-typed CDC lookalikes with exact `CDC::Core::ChangeEvent` and
|
|
22
|
+
`CDC::Core::TransactionEnvelope` objects.
|
|
23
|
+
- Enforced the PostgreSQL source output contract by rejecting non-core work
|
|
24
|
+
yielded by injected source adapters.
|
|
25
|
+
- Added an explicit persisted-payload deserialization boundary for sample JSON
|
|
26
|
+
and dead-letter replay.
|
|
27
|
+
- Moved configured batch accumulation and final partial-batch submission out of
|
|
28
|
+
`Mammoth::Application` and into the delivery runtime layer.
|
|
29
|
+
|
|
30
|
+
### Quality
|
|
31
|
+
|
|
32
|
+
- Added processor contract, observer, runtime notification, dispatch counter,
|
|
33
|
+
and Prometheus exposition coverage.
|
|
34
|
+
- Added exact core-type and persisted-payload round-trip coverage.
|
|
35
|
+
- Strengthened architecture boundary coverage with executable pgoutput adapter
|
|
36
|
+
output checks, downstream dependency guards, and an exact core-output RBS
|
|
37
|
+
contract for the PostgreSQL source.
|
|
38
|
+
- Added runtime batching ownership coverage so application orchestration cannot
|
|
39
|
+
reintroduce buffering or batch partitioning.
|
|
40
|
+
- Updated RBS signatures, YARD API documentation, examples, and benchmark
|
|
41
|
+
descriptions for the corrected processor/observer boundary.
|
|
42
|
+
|
|
5
43
|
## 0.8.0
|
|
6
44
|
|
|
7
45
|
### Changed
|
data/README.md
CHANGED
|
@@ -49,11 +49,12 @@ Mammoth OSS includes:
|
|
|
49
49
|
- SQLite operational memory bootstrap
|
|
50
50
|
- checkpoint persistence
|
|
51
51
|
- dead letter persistence
|
|
52
|
+
- delivered-envelope ledger persistence
|
|
52
53
|
- webhook delivery sink
|
|
53
54
|
- webhook fanout to multiple destinations
|
|
54
55
|
- fanout route filters by schema, table, and operation
|
|
55
56
|
- per-destination enable/disable and retry policy controls
|
|
56
|
-
- delivery worker with retry, checkpoint, and DLQ handling
|
|
57
|
+
- delivery worker with retry, delivered-ledger, checkpoint, and DLQ handling
|
|
57
58
|
- dead-letter inspection and filtered replay commands
|
|
58
59
|
- CDC-core event serialization boundary
|
|
59
60
|
- CDC Ecosystem source-adapter integration boundary
|
|
@@ -61,6 +62,7 @@ Mammoth OSS includes:
|
|
|
61
62
|
- public Helm chart support
|
|
62
63
|
- unit and e2e test tasks
|
|
63
64
|
- health and metrics endpoints
|
|
65
|
+
- canonical CDC dispatch counters through a `CDC::Core::Observer`
|
|
64
66
|
- explicit extension registries for state, destination, and runtime adapters
|
|
65
67
|
- node identity and local capability reporting
|
|
66
68
|
- lifecycle hooks, configuration providers, and reusable local command objects
|
|
@@ -70,8 +72,11 @@ Mammoth OSS includes:
|
|
|
70
72
|
Mammoth begins at CDC-core work items and ends at webhook fanout delivery.
|
|
71
73
|
|
|
72
74
|
Mammoth does not own pgoutput protocol parsing, value decoding, source
|
|
73
|
-
normalization,
|
|
74
|
-
|
|
75
|
+
normalization, or core dispatch vocabulary. Those belong to upstream CDC
|
|
76
|
+
Ecosystem components. Mammoth selects and composes a delivery runtime while
|
|
77
|
+
delegating its scheduling mechanics to the runtime layer. The runtime registry
|
|
78
|
+
wraps the selected adapter with configured batch accumulation; `Application`
|
|
79
|
+
only streams core work and coordinates lifecycle flush and shutdown calls.
|
|
75
80
|
|
|
76
81
|
For the live PostgreSQL stream, `pgoutput-source-adapter` incrementally owns
|
|
77
82
|
`Begin`/`Commit` buffering and emits exact `CDC::Core::ChangeEvent` or
|
|
@@ -79,6 +84,17 @@ For the live PostgreSQL stream, `pgoutput-source-adapter` incrementally owns
|
|
|
79
84
|
transport, parser, decoder, and source adapter and forwards the resulting core
|
|
80
85
|
work to delivery.
|
|
81
86
|
|
|
87
|
+
At the downstream boundary, `Mammoth::DeliveryProcessor` implements
|
|
88
|
+
`CDC::Core::Processor` and returns `CDC::Core::ProcessorResult`. Inline and
|
|
89
|
+
concurrent runtimes notify a `CDC::Core::Observer`; Mammoth's default observer
|
|
90
|
+
maps the canonical started, succeeded, failed, and skipped notifications to
|
|
91
|
+
Prometheus counters.
|
|
92
|
+
|
|
93
|
+
`Mammoth::ReplicationConsumer` accepts only exact core events and transaction
|
|
94
|
+
envelopes. Operator-facing JSON, such as `deliver-sample` input and persisted
|
|
95
|
+
dead letters, is reconstructed by `PersistedPayloadDeserializer` before it
|
|
96
|
+
re-enters delivery; stored hashes do not masquerade as live CDC work.
|
|
97
|
+
|
|
82
98
|
## Extensions
|
|
83
99
|
|
|
84
100
|
Mammoth OSS exposes small adapter registries for future extensions:
|
|
@@ -135,7 +151,8 @@ bundle exec ./exe/mammoth start config/mammoth.example.yml
|
|
|
135
151
|
bundle exec ./exe/mammoth observability config/mammoth.example.yml
|
|
136
152
|
```
|
|
137
153
|
|
|
138
|
-
|
|
154
|
+
Reconstruct and deliver a single persisted event JSON file through Mammoth's
|
|
155
|
+
core delivery path:
|
|
139
156
|
|
|
140
157
|
```bash
|
|
141
158
|
bundle exec ./exe/mammoth deliver-sample \
|
|
@@ -152,6 +169,10 @@ Mammoth stores operational memory in SQLite:
|
|
|
152
169
|
- `dead_letters`
|
|
153
170
|
- `delivered_envelopes`
|
|
154
171
|
|
|
172
|
+
SQLite is the built-in default behind `operational_state.adapter`. Bootstrap,
|
|
173
|
+
status, observability, and dead-letter commands consume the adapter contract
|
|
174
|
+
rather than opening SQLite directly.
|
|
175
|
+
|
|
155
176
|
## Performance
|
|
156
177
|
|
|
157
178
|
Mammoth includes local benchmarks for the product surfaces operators tune in
|
data/config/mammoth.schema.json
CHANGED
|
@@ -513,7 +513,7 @@
|
|
|
513
513
|
"batch_size": {
|
|
514
514
|
"type": "integer",
|
|
515
515
|
"minimum": 1,
|
|
516
|
-
"description": "Number of work units
|
|
516
|
+
"description": "Number of work units accumulated by the runtime layer before submission to the selected adapter. Values greater than 1 allow ordering-insensitive deliveries to overlap more visibly."
|
|
517
517
|
},
|
|
518
518
|
"preserve_order": {
|
|
519
519
|
"type": "boolean",
|
data/lib/mammoth/application.rb
CHANGED
|
@@ -4,24 +4,28 @@ module Mammoth
|
|
|
4
4
|
# Top-level Mammoth application runtime.
|
|
5
5
|
#
|
|
6
6
|
# Application wires Mammoth's delivery-side runtime pieces: configuration,
|
|
7
|
-
#
|
|
8
|
-
# store, dead
|
|
7
|
+
# operational-state adapter, replication consumer, delivery worker, checkpoint
|
|
8
|
+
# store, dead-letter store, and webhook sink. Upstream PostgreSQL transport
|
|
9
9
|
# composition stays outside this class so the application runtime consumes an
|
|
10
10
|
# injected CDC work source rather than owning upstream CDC source-adapter
|
|
11
11
|
# lifecycle decisions.
|
|
12
12
|
class Application
|
|
13
|
-
attr_reader :config, :state_adapter, :consumer, :delivery_worker, :checkpoint_store, :lifecycle_hooks
|
|
13
|
+
attr_reader :config, :state_adapter, :consumer, :delivery_worker, :checkpoint_store, :lifecycle_hooks, :observer
|
|
14
14
|
|
|
15
15
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
16
16
|
# @param source [#each, nil] injectable event source for tests and demos
|
|
17
17
|
# @param sink [#deliver, nil] optional destination sink
|
|
18
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
18
19
|
# @param sleeper [#call] retry sleep strategy
|
|
19
20
|
# @param lifecycle_hooks [Mammoth::LifecycleHooks, Hash] local lifecycle callbacks
|
|
20
|
-
|
|
21
|
+
# @param observer [CDC::Core::Observer, nil] dispatch lifecycle observer
|
|
22
|
+
def initialize(config, source: nil, sink: nil, state_adapter: nil, sleeper: Kernel.method(:sleep),
|
|
23
|
+
lifecycle_hooks: LifecycleHooks.new, observer: nil)
|
|
21
24
|
@config = config
|
|
22
25
|
@lifecycle_hooks = build_lifecycle_hooks(lifecycle_hooks)
|
|
23
|
-
@
|
|
24
|
-
@
|
|
26
|
+
@observer = observer || MetricsObserver.new
|
|
27
|
+
@state_adapter = state_adapter || build_state_adapter
|
|
28
|
+
@checkpoint_store = @state_adapter.checkpoint_store
|
|
25
29
|
@consumer = ReplicationConsumer.new(source: source || build_source, delivery_unit: delivery_unit)
|
|
26
30
|
@delivery_worker = sink ? build_delivery_worker(sink: sink, sleeper: sleeper) : build_configured_delivery_worker(sleeper:)
|
|
27
31
|
end
|
|
@@ -67,41 +71,18 @@ module Mammoth
|
|
|
67
71
|
|
|
68
72
|
def process_consumer(runtime)
|
|
69
73
|
processed = 0
|
|
70
|
-
batch = [nil].compact
|
|
71
74
|
|
|
72
75
|
consumer.start do |work|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
next unless batch.size >= runtime_batch_size
|
|
76
|
-
|
|
77
|
-
processed += process_batch(runtime, batch)
|
|
78
|
-
batch = []
|
|
79
|
-
else
|
|
80
|
-
process_work(runtime, work)
|
|
81
|
-
processed += 1
|
|
82
|
-
end
|
|
76
|
+
process_work(runtime, work)
|
|
77
|
+
processed += 1
|
|
83
78
|
end
|
|
84
79
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
def flush_batch(runtime, batch)
|
|
89
|
-
return 0 unless runtime_batching?(runtime) && batch.any?
|
|
90
|
-
|
|
91
|
-
process_batch(runtime, batch)
|
|
80
|
+
runtime.flush
|
|
81
|
+
processed
|
|
92
82
|
end
|
|
93
83
|
|
|
94
84
|
def process_work(runtime, work)
|
|
95
|
-
runtime.
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
def process_batch(runtime, batch)
|
|
99
|
-
runtime.process_many(batch)
|
|
100
|
-
batch.size
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
def runtime_batching?(runtime)
|
|
104
|
-
runtime && runtime_batch_size > 1
|
|
85
|
+
runtime.process(work)
|
|
105
86
|
end
|
|
106
87
|
|
|
107
88
|
def build_runtime
|
|
@@ -110,7 +91,9 @@ module Mammoth
|
|
|
110
91
|
processor: DeliveryProcessor.new(delivery_worker:, delivery_unit: delivery_unit),
|
|
111
92
|
concurrency: runtime_concurrency,
|
|
112
93
|
timeout: runtime_timeout,
|
|
113
|
-
preserve_order: runtime_preserve_order
|
|
94
|
+
preserve_order: runtime_preserve_order?,
|
|
95
|
+
batch_size: runtime_batch_size,
|
|
96
|
+
observer: observer
|
|
114
97
|
)
|
|
115
98
|
end
|
|
116
99
|
|
|
@@ -124,6 +107,7 @@ module Mammoth
|
|
|
124
107
|
sink: sink,
|
|
125
108
|
checkpoint_store: checkpoint_store,
|
|
126
109
|
dead_letter_store: state_adapter.dead_letter_store,
|
|
110
|
+
delivered_envelope_store: state_adapter.delivered_envelope_store,
|
|
127
111
|
sleeper: sleeper,
|
|
128
112
|
delivery_policy: delivery_policy
|
|
129
113
|
)
|
|
@@ -164,11 +148,7 @@ module Mammoth
|
|
|
164
148
|
end
|
|
165
149
|
|
|
166
150
|
def build_state_adapter
|
|
167
|
-
OperationalState::Registry.
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
def operational_state_adapter
|
|
171
|
-
config.dig("operational_state", "adapter") || "sqlite"
|
|
151
|
+
OperationalState::Registry.build_configured(config)
|
|
172
152
|
end
|
|
173
153
|
|
|
174
154
|
def delivery_unit
|
data/lib/mammoth/cli.rb
CHANGED
|
@@ -5,16 +5,6 @@ require "json"
|
|
|
5
5
|
module Mammoth
|
|
6
6
|
# Small command dispatcher for Mammoth's operator-facing CLI.
|
|
7
7
|
class CLI
|
|
8
|
-
# Internal replay envelope used for transaction dead-letter recovery.
|
|
9
|
-
DEAD_LETTER_TRANSACTION_ENVELOPE = Data.define(
|
|
10
|
-
:events,
|
|
11
|
-
:transaction_id,
|
|
12
|
-
:source_position,
|
|
13
|
-
:commit_lsn,
|
|
14
|
-
:committed_at,
|
|
15
|
-
:metadata
|
|
16
|
-
)
|
|
17
|
-
|
|
18
8
|
# Human-readable command usage printed for invalid or incomplete invocations.
|
|
19
9
|
USAGE = [
|
|
20
10
|
"Usage:",
|
|
@@ -94,9 +84,7 @@ module Mammoth
|
|
|
94
84
|
end
|
|
95
85
|
|
|
96
86
|
def status
|
|
97
|
-
|
|
98
|
-
store = SQLiteStore.connect(config.dig("sqlite", "path"))
|
|
99
|
-
Commands::StatusCommand.new(config, sqlite_store: store).call
|
|
87
|
+
Commands::StatusCommand.new(load_config).call
|
|
100
88
|
end
|
|
101
89
|
|
|
102
90
|
def start
|
|
@@ -2,23 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
module Mammoth
|
|
4
4
|
module Commands
|
|
5
|
-
# Initializes the configured operational
|
|
5
|
+
# Initializes the configured operational-state backend.
|
|
6
6
|
class BootstrapCommand
|
|
7
|
-
attr_reader :config, :output
|
|
7
|
+
attr_reader :config, :state_adapter, :output
|
|
8
8
|
|
|
9
9
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
10
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
10
11
|
# @param output [#puts] output stream
|
|
11
|
-
def initialize(config, output: $stdout)
|
|
12
|
+
def initialize(config, state_adapter: nil, output: $stdout)
|
|
12
13
|
@config = config
|
|
14
|
+
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
13
15
|
@output = output
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
# @return [Integer] process-style status code
|
|
17
19
|
def call
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
output.puts "
|
|
21
|
-
output.puts "
|
|
20
|
+
state_adapter.bootstrap!
|
|
21
|
+
summary = state_adapter.summary
|
|
22
|
+
output.puts "Operational state initialized"
|
|
23
|
+
output.puts "Adapter: #{summary.fetch(:adapter)}"
|
|
24
|
+
output.puts "Path: #{summary.fetch(:path)}" if summary.key?(:path)
|
|
25
|
+
output.puts "Tables: #{Array(summary.fetch(:tables)).join(", ")}" if summary.key?(:tables)
|
|
22
26
|
0
|
|
23
27
|
end
|
|
24
28
|
end
|
|
@@ -4,18 +4,20 @@ module Mammoth
|
|
|
4
4
|
module Commands
|
|
5
5
|
# Reusable local command object for dead-letter inspection and replay.
|
|
6
6
|
class DeadLettersCommand
|
|
7
|
-
attr_reader :argv, :lifecycle_hooks
|
|
7
|
+
attr_reader :argv, :state_adapter, :lifecycle_hooks
|
|
8
8
|
|
|
9
9
|
# @param argv [Array<String>] dead-letters CLI-style arguments
|
|
10
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
10
11
|
# @param lifecycle_hooks [Mammoth::LifecycleHooks, Hash] local lifecycle callbacks
|
|
11
|
-
def initialize(argv, lifecycle_hooks: LifecycleHooks.new)
|
|
12
|
+
def initialize(argv, state_adapter: nil, lifecycle_hooks: LifecycleHooks.new)
|
|
12
13
|
@argv = argv
|
|
14
|
+
@state_adapter = state_adapter
|
|
13
15
|
@lifecycle_hooks = lifecycle_hooks
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
# @return [Integer] process-style status code
|
|
17
19
|
def call
|
|
18
|
-
DeadLetterCommands.call(argv, lifecycle_hooks: lifecycle_hooks)
|
|
20
|
+
DeadLetterCommands.call(argv, state_adapter: state_adapter, lifecycle_hooks: lifecycle_hooks)
|
|
19
21
|
end
|
|
20
22
|
end
|
|
21
23
|
end
|
|
@@ -23,7 +23,7 @@ module Mammoth
|
|
|
23
23
|
def call
|
|
24
24
|
raise ConfigurationError, "event JSON file not found: #{event_path}" unless File.file?(event_path)
|
|
25
25
|
|
|
26
|
-
event = JSON.parse(File.read(event_path))
|
|
26
|
+
event = PersistedPayloadDeserializer.event(JSON.parse(File.read(event_path)))
|
|
27
27
|
processed = Application.new(config, source: [event], lifecycle_hooks: lifecycle_hooks).start
|
|
28
28
|
output.puts "Processed sample events: #{processed}"
|
|
29
29
|
0
|
|
@@ -5,14 +5,14 @@ module Mammoth
|
|
|
5
5
|
module Commands
|
|
6
6
|
# Reusable local command object for status inspection.
|
|
7
7
|
class StatusCommand
|
|
8
|
-
attr_reader :config, :
|
|
8
|
+
attr_reader :config, :state_adapter, :output
|
|
9
9
|
|
|
10
10
|
# @param config [Mammoth::Configuration] loaded configuration
|
|
11
|
-
# @param
|
|
11
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
12
12
|
# @param output [#puts] output stream
|
|
13
|
-
def initialize(config,
|
|
13
|
+
def initialize(config, state_adapter: nil, output: $stdout)
|
|
14
14
|
@config = config
|
|
15
|
-
@
|
|
15
|
+
@state_adapter = state_adapter || OperationalState::Registry.build_configured(config)
|
|
16
16
|
@output = output
|
|
17
17
|
end
|
|
18
18
|
|
|
@@ -20,7 +20,7 @@ module Mammoth
|
|
|
20
20
|
#
|
|
21
21
|
# @return [Integer] process-style status code
|
|
22
22
|
def call
|
|
23
|
-
Status.call(config,
|
|
23
|
+
Status.call(config, state_adapter: state_adapter, output: output)
|
|
24
24
|
0
|
|
25
25
|
end
|
|
26
26
|
end
|
|
@@ -7,17 +7,19 @@ module Mammoth
|
|
|
7
7
|
# I/O fan-out to cdc-concurrent. This class is intentionally small so the
|
|
8
8
|
# runtime boundary remains easy to test and replace.
|
|
9
9
|
class ConcurrentDeliveryRuntime
|
|
10
|
-
attr_reader :processor, :concurrency, :timeout, :preserve_order, :pool
|
|
10
|
+
attr_reader :processor, :concurrency, :timeout, :preserve_order, :pool, :observer
|
|
11
11
|
|
|
12
12
|
# @param processor [#process] delivery processor
|
|
13
13
|
# @param concurrency [Integer] number of concurrent delivery workers
|
|
14
14
|
# @param timeout [Numeric, nil] optional per-item timeout
|
|
15
15
|
# @param preserve_order [Boolean] preserve output order when supported
|
|
16
|
-
|
|
16
|
+
# @param observer [CDC::Core::Observer] dispatch lifecycle observer
|
|
17
|
+
def initialize(processor:, concurrency:, timeout:, preserve_order:, observer: CDC::Core::Observer.new)
|
|
17
18
|
@processor = processor
|
|
18
19
|
@concurrency = concurrency
|
|
19
20
|
@timeout = timeout
|
|
20
21
|
@preserve_order = preserve_order
|
|
22
|
+
@observer = observer
|
|
21
23
|
@pool = build_pool
|
|
22
24
|
end
|
|
23
25
|
|
|
@@ -28,7 +30,10 @@ module Mammoth
|
|
|
28
30
|
def process_many(items)
|
|
29
31
|
return [] if items.empty?
|
|
30
32
|
|
|
31
|
-
|
|
33
|
+
items.each { |item| observer.dispatch_started(item) }
|
|
34
|
+
pool.process_many(items).tap do |results|
|
|
35
|
+
results.each { |result| observe_result(result) }
|
|
36
|
+
end
|
|
32
37
|
end
|
|
33
38
|
|
|
34
39
|
# Shutdown the underlying runtime when supported.
|
|
@@ -41,6 +46,13 @@ module Mammoth
|
|
|
41
46
|
|
|
42
47
|
private
|
|
43
48
|
|
|
49
|
+
def observe_result(result)
|
|
50
|
+
return observer.dispatch_succeeded(result) if result.success?
|
|
51
|
+
return observer.dispatch_failed(result) if result.failure?
|
|
52
|
+
|
|
53
|
+
observer.dispatch_skipped(result)
|
|
54
|
+
end
|
|
55
|
+
|
|
44
56
|
def build_pool
|
|
45
57
|
require "cdc/concurrent"
|
|
46
58
|
CDC::Concurrent::ProcessorPool.new(processor:, concurrency:, timeout:, preserve_order:)
|
|
@@ -7,27 +7,19 @@ module Mammoth
|
|
|
7
7
|
# Operator commands for inspecting and replaying dead letters.
|
|
8
8
|
# rubocop:disable Metrics/ClassLength
|
|
9
9
|
class DeadLetterCommands
|
|
10
|
-
# Internal replay envelope used for transaction dead-letter recovery.
|
|
11
|
-
DEAD_LETTER_TRANSACTION_ENVELOPE = Data.define(
|
|
12
|
-
:events,
|
|
13
|
-
:transaction_id,
|
|
14
|
-
:source_position,
|
|
15
|
-
:commit_lsn,
|
|
16
|
-
:committed_at,
|
|
17
|
-
:metadata
|
|
18
|
-
)
|
|
19
|
-
|
|
20
10
|
attr_reader :argv, :lifecycle_hooks
|
|
21
11
|
|
|
22
12
|
# @param argv [Array<String>] command line arguments
|
|
23
|
-
def self.call(argv, lifecycle_hooks: LifecycleHooks.new)
|
|
24
|
-
new(argv, lifecycle_hooks: lifecycle_hooks).call
|
|
13
|
+
def self.call(argv, state_adapter: nil, lifecycle_hooks: LifecycleHooks.new)
|
|
14
|
+
new(argv, state_adapter: state_adapter, lifecycle_hooks: lifecycle_hooks).call
|
|
25
15
|
end
|
|
26
16
|
|
|
27
17
|
# @param argv [Array<String>] command line arguments
|
|
18
|
+
# @param state_adapter [Mammoth::OperationalState::Adapter, nil] operational state dependency
|
|
28
19
|
# @param lifecycle_hooks [Mammoth::LifecycleHooks, Hash] local lifecycle callbacks
|
|
29
|
-
def initialize(argv, lifecycle_hooks: LifecycleHooks.new)
|
|
20
|
+
def initialize(argv, state_adapter: nil, lifecycle_hooks: LifecycleHooks.new)
|
|
30
21
|
@argv = argv
|
|
22
|
+
@state_adapter = state_adapter
|
|
31
23
|
@lifecycle_hooks = lifecycle_hooks.is_a?(LifecycleHooks) ? lifecycle_hooks : LifecycleHooks.new(lifecycle_hooks)
|
|
32
24
|
end
|
|
33
25
|
|
|
@@ -98,11 +90,15 @@ module Mammoth
|
|
|
98
90
|
end
|
|
99
91
|
|
|
100
92
|
def dead_letter_store
|
|
101
|
-
@dead_letter_store ||=
|
|
93
|
+
@dead_letter_store ||= state_adapter.dead_letter_store
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def state_adapter
|
|
97
|
+
@state_adapter ||= OperationalState::Registry.build_configured(load_config)
|
|
102
98
|
end
|
|
103
99
|
|
|
104
100
|
def worker
|
|
105
|
-
@worker ||= Application.new(load_config).delivery_worker
|
|
101
|
+
@worker ||= Application.new(load_config, state_adapter: state_adapter).delivery_worker
|
|
106
102
|
end
|
|
107
103
|
|
|
108
104
|
def dead_letter_id
|
|
@@ -250,9 +246,10 @@ module Mammoth
|
|
|
250
246
|
end
|
|
251
247
|
|
|
252
248
|
def replay_event(destination_name, payload)
|
|
253
|
-
|
|
249
|
+
event = PersistedPayloadDeserializer.event(payload)
|
|
250
|
+
return worker.deliver_to(destination_name, event) if worker.respond_to?(:deliver_to)
|
|
254
251
|
|
|
255
|
-
worker.deliver(
|
|
252
|
+
worker.deliver(event)
|
|
256
253
|
end
|
|
257
254
|
|
|
258
255
|
def replay_transaction(destination_name, envelope)
|
|
@@ -270,14 +267,7 @@ module Mammoth
|
|
|
270
267
|
end
|
|
271
268
|
|
|
272
269
|
def transaction_envelope(payload)
|
|
273
|
-
|
|
274
|
-
payload.fetch("events"),
|
|
275
|
-
payload.fetch("transaction_id"),
|
|
276
|
-
payload["source_position"],
|
|
277
|
-
payload["commit_lsn"],
|
|
278
|
-
payload["committed_at"],
|
|
279
|
-
payload["metadata"] || {}
|
|
280
|
-
)
|
|
270
|
+
PersistedPayloadDeserializer.transaction(payload)
|
|
281
271
|
end
|
|
282
272
|
|
|
283
273
|
def show_payload(row)
|
|
@@ -315,6 +305,7 @@ module Mammoth
|
|
|
315
305
|
def replay_context(extra = {})
|
|
316
306
|
{
|
|
317
307
|
config: load_config,
|
|
308
|
+
state_adapter: state_adapter,
|
|
318
309
|
dead_letter_store: dead_letter_store,
|
|
319
310
|
delivery_worker: worker
|
|
320
311
|
}.merge(extra)
|
|
@@ -15,7 +15,7 @@ module Mammoth
|
|
|
15
15
|
|
|
16
16
|
# Store a failed delivery.
|
|
17
17
|
#
|
|
18
|
-
# @param event [
|
|
18
|
+
# @param event [CDC::Core::ChangeEvent, CDC::Core::TransactionEnvelope] normalized core work item
|
|
19
19
|
# @param destination_name [String] destination name
|
|
20
20
|
# @param error [Exception, nil] delivery failure
|
|
21
21
|
# @param retry_count [Integer] number of delivery attempts
|
|
@@ -6,7 +6,7 @@ module Mammoth
|
|
|
6
6
|
# The processor keeps cdc-concurrent integration narrow: cdc-concurrent owns
|
|
7
7
|
# I/O-heavy fan-out mechanics, while DeliveryWorker owns Mammoth relay
|
|
8
8
|
# semantics such as retries, dead letters, and checkpoint writes.
|
|
9
|
-
class DeliveryProcessor
|
|
9
|
+
class DeliveryProcessor < CDC::Core::Processor
|
|
10
10
|
@concurrent_safe = false
|
|
11
11
|
|
|
12
12
|
class << self
|
|
@@ -37,6 +37,7 @@ module Mammoth
|
|
|
37
37
|
# @param delivery_worker [Mammoth::DeliveryWorker] relay-aware delivery worker
|
|
38
38
|
# @param delivery_unit [String, Symbol] event or transaction
|
|
39
39
|
def initialize(delivery_worker:, delivery_unit: :event)
|
|
40
|
+
super()
|
|
40
41
|
@delivery_worker = delivery_worker
|
|
41
42
|
@delivery_unit = delivery_unit.to_sym
|
|
42
43
|
end
|
|
@@ -51,7 +52,7 @@ module Mammoth
|
|
|
51
52
|
# Process one work item from CDC::Concurrent::ProcessorPool.
|
|
52
53
|
#
|
|
53
54
|
# @param work [Object] event or transaction envelope
|
|
54
|
-
# @return [
|
|
55
|
+
# @return [CDC::Core::ProcessorResult] normalized processor result
|
|
55
56
|
def call(work)
|
|
56
57
|
process(work)
|
|
57
58
|
end
|
|
@@ -59,14 +60,52 @@ module Mammoth
|
|
|
59
60
|
# Process one work item using the configured delivery unit.
|
|
60
61
|
#
|
|
61
62
|
# @param work [Object] event or transaction envelope
|
|
62
|
-
# @return [
|
|
63
|
+
# @return [CDC::Core::ProcessorResult] normalized processor result
|
|
63
64
|
def process(work)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
65
|
+
summary = deliver(work)
|
|
66
|
+
build_result(work, summary)
|
|
67
|
+
rescue StandardError => e
|
|
68
|
+
failure_result(work, e, retryable: e.is_a?(DeliveryError))
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def deliver(work)
|
|
74
|
+
return delivery_worker.deliver_transaction(work) if delivery_unit == :transaction
|
|
75
|
+
|
|
76
|
+
delivery_worker.deliver(work)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def build_result(work, summary)
|
|
80
|
+
case summary[:status].to_s
|
|
81
|
+
when "skipped"
|
|
82
|
+
CDC::Core::ProcessorResult.skipped(work, metadata: result_metadata(summary))
|
|
83
|
+
when "dead_lettered", "fanout_partial"
|
|
84
|
+
failure_result(work, DeliveryError.new(failure_reason(summary)), retryable: false, summary: summary)
|
|
67
85
|
else
|
|
68
|
-
|
|
86
|
+
CDC::Core::ProcessorResult.success(work, value: summary, metadata: result_metadata(summary))
|
|
69
87
|
end
|
|
70
88
|
end
|
|
89
|
+
|
|
90
|
+
def failure_result(work, error, retryable:, summary: nil)
|
|
91
|
+
CDC::Core::ProcessorResult.failure(
|
|
92
|
+
error,
|
|
93
|
+
event: work,
|
|
94
|
+
reason: error.message,
|
|
95
|
+
retryable: retryable,
|
|
96
|
+
processor: self.class.name,
|
|
97
|
+
metadata: result_metadata(summary)
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def failure_reason(summary)
|
|
102
|
+
"delivery completed with #{summary.fetch(:status)} status"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def result_metadata(summary)
|
|
106
|
+
metadata = { processor: self.class.name, delivery_unit: delivery_unit.to_s }
|
|
107
|
+
metadata[:delivery] = summary if summary
|
|
108
|
+
metadata
|
|
109
|
+
end
|
|
71
110
|
end
|
|
72
111
|
end
|