mammoth 0.7.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +55 -0
- data/README.md +38 -7
- data/config/mammoth.schema.json +1 -1
- data/lib/mammoth/application.rb +20 -40
- data/lib/mammoth/cli.rb +1 -13
- data/lib/mammoth/commands/bootstrap_command.rb +11 -7
- data/lib/mammoth/commands/dead_letters_command.rb +5 -3
- data/lib/mammoth/commands/deliver_sample_command.rb +1 -1
- data/lib/mammoth/commands/status_command.rb +5 -5
- data/lib/mammoth/concurrent_delivery_runtime.rb +15 -3
- data/lib/mammoth/dead_letter_commands.rb +16 -25
- data/lib/mammoth/dead_letter_store.rb +1 -1
- data/lib/mammoth/delivery_processor.rb +46 -7
- data/lib/mammoth/delivery_worker.rb +10 -8
- data/lib/mammoth/dispatch_metrics.rb +55 -0
- data/lib/mammoth/event_serializer.rb +23 -12
- data/lib/mammoth/fanout_delivery_worker.rb +4 -4
- data/lib/mammoth/metrics_observer.rb +52 -0
- data/lib/mammoth/observability_metrics.rb +69 -0
- data/lib/mammoth/observability_server.rb +6 -6
- data/lib/mammoth/observability_snapshot.rb +59 -60
- data/lib/mammoth/operational_state/adapter.rb +16 -0
- data/lib/mammoth/operational_state/registry.rb +8 -0
- data/lib/mammoth/operational_state/sqlite_adapter.rb +18 -2
- data/lib/mammoth/persisted_payload_deserializer.rb +85 -0
- data/lib/mammoth/replication_consumer.rb +15 -47
- data/lib/mammoth/runtimes/batching_runtime.rb +59 -0
- data/lib/mammoth/runtimes/concurrent_adapter.rb +4 -2
- data/lib/mammoth/runtimes/inline_adapter.rb +19 -5
- data/lib/mammoth/runtimes/registry.rb +3 -2
- data/lib/mammoth/sources/postgres.rb +39 -129
- data/lib/mammoth/status.rb +22 -18
- data/lib/mammoth/transaction_envelope_serializer.rb +13 -15
- data/lib/mammoth/version.rb +1 -1
- data/lib/mammoth/webhook_sink.rb +2 -2
- data/lib/mammoth.rb +8 -1
- metadata +14 -3
|
@@ -15,16 +15,20 @@ module Mammoth
|
|
|
15
15
|
# Default payload type for transaction webhook delivery.
|
|
16
16
|
PAYLOAD_TYPE = "transaction.committed"
|
|
17
17
|
|
|
18
|
-
# Serialize a CDC::Core::TransactionEnvelope
|
|
18
|
+
# Serialize a CDC::Core::TransactionEnvelope into a Hash.
|
|
19
19
|
#
|
|
20
|
-
# @param envelope [
|
|
20
|
+
# @param envelope [CDC::Core::TransactionEnvelope] transaction envelope
|
|
21
21
|
# @return [Hash] webhook-ready transaction payload
|
|
22
22
|
def self.call(envelope)
|
|
23
23
|
new(envelope).call
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
# @param envelope [
|
|
26
|
+
# @param envelope [CDC::Core::TransactionEnvelope] transaction envelope
|
|
27
27
|
def initialize(envelope)
|
|
28
|
+
unless envelope.is_a?(CDC::Core::TransactionEnvelope)
|
|
29
|
+
raise ArgumentError, "envelope must be a CDC::Core::TransactionEnvelope"
|
|
30
|
+
end
|
|
31
|
+
|
|
28
32
|
@envelope = envelope
|
|
29
33
|
end
|
|
30
34
|
|
|
@@ -34,7 +38,7 @@ module Mammoth
|
|
|
34
38
|
def call
|
|
35
39
|
event_payloads = envelope.events.map { |event| EventSerializer.call(event) }
|
|
36
40
|
{
|
|
37
|
-
"event_id" =>
|
|
41
|
+
"event_id" => envelope_metadata["event_id"] || SecureRandom.uuid,
|
|
38
42
|
"type" => PAYLOAD_TYPE,
|
|
39
43
|
"source" => first_event_value(event_payloads, "source") || EventSerializer::DEFAULT_SOURCE,
|
|
40
44
|
"transaction_id" => envelope.transaction_id,
|
|
@@ -43,7 +47,7 @@ module Mammoth
|
|
|
43
47
|
"committed_at" => committed_at,
|
|
44
48
|
"event_count" => event_payloads.length,
|
|
45
49
|
"events" => event_payloads,
|
|
46
|
-
"metadata" =>
|
|
50
|
+
"metadata" => envelope_metadata
|
|
47
51
|
}
|
|
48
52
|
end
|
|
49
53
|
|
|
@@ -58,27 +62,21 @@ module Mammoth
|
|
|
58
62
|
|
|
59
63
|
attr_reader :envelope
|
|
60
64
|
|
|
61
|
-
def envelope_hash
|
|
62
|
-
@envelope_hash ||= envelope.respond_to?(:to_h) ? stringify_keys(envelope.to_h) : {}
|
|
63
|
-
end
|
|
64
|
-
|
|
65
65
|
def stringify_keys(hash)
|
|
66
66
|
hash.to_h.transform_keys(&:to_s)
|
|
67
67
|
end
|
|
68
68
|
|
|
69
|
-
def
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
envelope_hash[key]
|
|
69
|
+
def envelope_metadata
|
|
70
|
+
@envelope_metadata ||= stringify_keys(envelope.metadata)
|
|
73
71
|
end
|
|
74
72
|
|
|
75
73
|
def source_position(event_payloads)
|
|
76
|
-
|
|
74
|
+
envelope.commit_lsn ||
|
|
77
75
|
first_event_value(event_payloads.reverse, "source_position")
|
|
78
76
|
end
|
|
79
77
|
|
|
80
78
|
def committed_at
|
|
81
|
-
value =
|
|
79
|
+
value = envelope.committed_at
|
|
82
80
|
return value.iso8601 if value.respond_to?(:iso8601)
|
|
83
81
|
|
|
84
82
|
value || Time.now.utc.iso8601
|
data/lib/mammoth/version.rb
CHANGED
data/lib/mammoth/webhook_sink.rb
CHANGED
|
@@ -94,7 +94,7 @@ module Mammoth
|
|
|
94
94
|
|
|
95
95
|
# Deliver an event to the webhook endpoint.
|
|
96
96
|
#
|
|
97
|
-
# @param event [
|
|
97
|
+
# @param event [CDC::Core::ChangeEvent] normalized event
|
|
98
98
|
# @return [Hash] delivery result
|
|
99
99
|
# @raise [Mammoth::DeliveryError] when delivery fails
|
|
100
100
|
def deliver(event)
|
|
@@ -103,7 +103,7 @@ module Mammoth
|
|
|
103
103
|
|
|
104
104
|
# Deliver a transaction envelope to the webhook endpoint.
|
|
105
105
|
#
|
|
106
|
-
# @param envelope [
|
|
106
|
+
# @param envelope [CDC::Core::TransactionEnvelope] CDC transaction envelope
|
|
107
107
|
# @return [Hash] delivery result
|
|
108
108
|
# @raise [Mammoth::DeliveryError] when delivery fails
|
|
109
109
|
def deliver_transaction(envelope)
|
data/lib/mammoth.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "cdc/core"
|
|
4
|
+
|
|
3
5
|
require_relative "mammoth/version"
|
|
4
6
|
require_relative "mammoth/errors"
|
|
5
7
|
require_relative "mammoth/registry"
|
|
@@ -14,6 +16,9 @@ require_relative "mammoth/commands/status_command"
|
|
|
14
16
|
require_relative "mammoth/commands/start_command"
|
|
15
17
|
require_relative "mammoth/commands/deliver_sample_command"
|
|
16
18
|
require_relative "mammoth/commands/dead_letters_command"
|
|
19
|
+
require_relative "mammoth/dispatch_metrics"
|
|
20
|
+
require_relative "mammoth/metrics_observer"
|
|
21
|
+
require_relative "mammoth/observability_metrics"
|
|
17
22
|
require_relative "mammoth/observability_snapshot"
|
|
18
23
|
require_relative "mammoth/observability_server"
|
|
19
24
|
require_relative "mammoth/sqlite_store"
|
|
@@ -22,6 +27,7 @@ require_relative "mammoth/dead_letter_store"
|
|
|
22
27
|
require_relative "mammoth/delivered_envelope_store"
|
|
23
28
|
require_relative "mammoth/event_serializer"
|
|
24
29
|
require_relative "mammoth/transaction_envelope_serializer"
|
|
30
|
+
require_relative "mammoth/persisted_payload_deserializer"
|
|
25
31
|
require_relative "mammoth/route_filter"
|
|
26
32
|
require_relative "mammoth/webhook_sink"
|
|
27
33
|
require_relative "mammoth/operational_state/adapter"
|
|
@@ -37,6 +43,7 @@ require_relative "mammoth/concurrent_delivery_runtime"
|
|
|
37
43
|
require_relative "mammoth/runtimes/adapter"
|
|
38
44
|
require_relative "mammoth/runtimes/inline_adapter"
|
|
39
45
|
require_relative "mammoth/runtimes/concurrent_adapter"
|
|
46
|
+
require_relative "mammoth/runtimes/batching_runtime"
|
|
40
47
|
require_relative "mammoth/runtimes/registry"
|
|
41
48
|
require_relative "mammoth/sources/postgres"
|
|
42
49
|
require_relative "mammoth/cdc_source"
|
|
@@ -47,7 +54,7 @@ require_relative "mammoth/cli"
|
|
|
47
54
|
|
|
48
55
|
# Mammoth is a self-hosted PostgreSQL event relay.
|
|
49
56
|
#
|
|
50
|
-
# Mammoth 0.
|
|
57
|
+
# Mammoth 0.8.x is a single-node PostgreSQL CDC relay with webhook fanout,
|
|
51
58
|
# replayable operational SQLite state, health/metrics endpoints, and explicit
|
|
52
59
|
# extension contracts for state, destination, runtime, lifecycle, configuration,
|
|
53
60
|
# and local command integrations.
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mammoth
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.9.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ken C. Demanawa
|
|
@@ -99,14 +99,20 @@ dependencies:
|
|
|
99
99
|
requirements:
|
|
100
100
|
- - "~>"
|
|
101
101
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: '0.
|
|
102
|
+
version: '0.2'
|
|
103
|
+
- - ">="
|
|
104
|
+
- !ruby/object:Gem::Version
|
|
105
|
+
version: 0.2.0
|
|
103
106
|
type: :runtime
|
|
104
107
|
prerelease: false
|
|
105
108
|
version_requirements: !ruby/object:Gem::Requirement
|
|
106
109
|
requirements:
|
|
107
110
|
- - "~>"
|
|
108
111
|
- !ruby/object:Gem::Version
|
|
109
|
-
version: '0.
|
|
112
|
+
version: '0.2'
|
|
113
|
+
- - ">="
|
|
114
|
+
- !ruby/object:Gem::Version
|
|
115
|
+
version: 0.2.0
|
|
110
116
|
- !ruby/object:Gem::Dependency
|
|
111
117
|
name: sqlite3
|
|
112
118
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -179,20 +185,25 @@ files:
|
|
|
179
185
|
- lib/mammoth/destinations/adapter.rb
|
|
180
186
|
- lib/mammoth/destinations/registry.rb
|
|
181
187
|
- lib/mammoth/destinations/webhook_adapter.rb
|
|
188
|
+
- lib/mammoth/dispatch_metrics.rb
|
|
182
189
|
- lib/mammoth/errors.rb
|
|
183
190
|
- lib/mammoth/event_serializer.rb
|
|
184
191
|
- lib/mammoth/fanout_delivery_worker.rb
|
|
185
192
|
- lib/mammoth/lifecycle_hooks.rb
|
|
193
|
+
- lib/mammoth/metrics_observer.rb
|
|
186
194
|
- lib/mammoth/node_identity.rb
|
|
195
|
+
- lib/mammoth/observability_metrics.rb
|
|
187
196
|
- lib/mammoth/observability_server.rb
|
|
188
197
|
- lib/mammoth/observability_snapshot.rb
|
|
189
198
|
- lib/mammoth/operational_state/adapter.rb
|
|
190
199
|
- lib/mammoth/operational_state/registry.rb
|
|
191
200
|
- lib/mammoth/operational_state/sqlite_adapter.rb
|
|
201
|
+
- lib/mammoth/persisted_payload_deserializer.rb
|
|
192
202
|
- lib/mammoth/registry.rb
|
|
193
203
|
- lib/mammoth/replication_consumer.rb
|
|
194
204
|
- lib/mammoth/route_filter.rb
|
|
195
205
|
- lib/mammoth/runtimes/adapter.rb
|
|
206
|
+
- lib/mammoth/runtimes/batching_runtime.rb
|
|
196
207
|
- lib/mammoth/runtimes/concurrent_adapter.rb
|
|
197
208
|
- lib/mammoth/runtimes/inline_adapter.rb
|
|
198
209
|
- lib/mammoth/runtimes/registry.rb
|