pgoutput-decoder 0.0.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -2
- data/README.md +6 -1
- data/lib/pgoutput/decoder/errors.rb +20 -0
- data/lib/pgoutput/decoder/events.rb +50 -0
- data/lib/pgoutput/decoder/relation_cache.rb +34 -0
- data/lib/pgoutput/decoder/row_builder.rb +44 -0
- data/lib/pgoutput/decoder/type_registry.rb +171 -0
- data/lib/pgoutput/decoder/value_decoder.rb +34 -0
- data/lib/pgoutput/decoder/version.rb +3 -2
- data/lib/pgoutput/decoder.rb +167 -3
- data/lib/pgoutput_decoder.rb +3 -0
- data/sig/pgoutput_decoder.rbs +165 -0
- metadata +16 -93
- data/sig/pgoutput/decoder.rbs +0 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a1a32b04410b404b4eb936eac7e3cae1548830c13cdbe7a1eb5e1604fb2bdabe
|
|
4
|
+
data.tar.gz: b14c651c88ea56ef70b9674def89cfd774de17c1efaf9894544e6d5212061045
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: aa2a36f348bab0d37668e40909acefc4b101e38a8ee990ef217848087eaaa5b3473741e33238a5fff6106e660138a946451f42ba9db25ca069ee49117cc72973
|
|
7
|
+
data.tar.gz: a965ef38835a5134b0ebd7f9a5eca4a54bf3786421f8365fc9110907088b797a7d2d8e86b14d2321ed37af4633df1f03ccf68b317d9209a0b43d94adc64606ee
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,49 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
1
3
|
## [Unreleased]
|
|
2
4
|
|
|
3
|
-
##
|
|
5
|
+
## 0.1.1 - 2026-06-17
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
* Curated RBS signatures shipped with the gem.
|
|
10
|
+
* Fixed duplicate `Pgoutput::Decoder` type declarations that caused Steep/RBS environment loading failures.
|
|
11
|
+
* Corrected decoder type signatures to align with actual runtime behavior.
|
|
12
|
+
* Improved `TypeRegistry` type annotations and value narrowing.
|
|
13
|
+
* Fixed Steep type-checking issues around decoder lookup, JSON decoding, UUID decoding, and numeric value decoding.
|
|
14
|
+
* Added missing standard library type dependencies required by Steep.
|
|
15
|
+
* Improved compatibility with downstream consumers using:
|
|
16
|
+
|
|
17
|
+
* `library "pgoutput-decoder"`
|
|
18
|
+
* `bundle exec steep check`
|
|
19
|
+
|
|
20
|
+
### Documentation
|
|
21
|
+
|
|
22
|
+
* Refined shipped type definitions to better reflect the public API surface.
|
|
23
|
+
|
|
24
|
+
### Internal
|
|
25
|
+
|
|
26
|
+
* No runtime behavior changes.
|
|
27
|
+
* No protocol decoding changes.
|
|
28
|
+
* No public API changes.
|
|
29
|
+
* This release focuses on RBS, Steep, and developer tooling correctness.
|
|
30
|
+
|
|
31
|
+
## [0.1.0] - 2026-06-01
|
|
32
|
+
|
|
33
|
+
### Added
|
|
4
34
|
|
|
5
|
-
-
|
|
35
|
+
- Added initial `pgoutput-decoder` gem structure.
|
|
36
|
+
- Added dependency on `pgoutput-parser`.
|
|
37
|
+
- Added `Pgoutput::Decoder` facade.
|
|
38
|
+
- Added decoded event models for Begin, Commit, Insert, Update, and Delete.
|
|
39
|
+
- Added relation cache for parser Relation messages.
|
|
40
|
+
- Added active transaction tracking using Begin message XID.
|
|
41
|
+
- Added decoded row hash construction from relation columns and tuple values.
|
|
42
|
+
- Added default PostgreSQL OID decoders for common scalar types.
|
|
43
|
+
- Added conservative binary decoding for fixed-width scalar values.
|
|
44
|
+
- Added custom OID decoder support.
|
|
45
|
+
- Added Ractor-shareable decoded event outputs.
|
|
46
|
+
- Added Minitest coverage for registry, value decoding, row building, relation cache, and integration flows.
|
|
47
|
+
- Added RBS signatures.
|
|
48
|
+
- Added README documentation.
|
|
49
|
+
- Added CI and release workflow templates.
|
data/README.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# pgoutput-decoder
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/pgoutput-decoder)
|
|
4
|
+
[](https://github.com/kanutocd/pgoutput-decoder/actions)
|
|
5
|
+
[](https://www.ruby-lang.org/en/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
3
8
|
A high-level PostgreSQL `pgoutput` logical replication value decoder for Ruby.
|
|
4
9
|
|
|
5
10
|
`pgoutput-decoder` is the companion layer to [`pgoutput-parser`](https://rubygems.org/gems/pgoutput-parser). It accepts immutable protocol messages produced by `pgoutput-parser` and turns tuple payloads into application-friendly Ruby row-change events.
|
|
@@ -275,4 +280,4 @@ bundle exec steep check
|
|
|
275
280
|
|
|
276
281
|
## License
|
|
277
282
|
|
|
278
|
-
MIT.
|
|
283
|
+
[MIT](LICENSE.txt).
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Base decoder error.
|
|
6
|
+
class Error < StandardError; end
|
|
7
|
+
|
|
8
|
+
# Raised when a DML message references an unknown relation.
|
|
9
|
+
class UnknownRelationError < Error; end
|
|
10
|
+
|
|
11
|
+
# Raised when a message type cannot be decoded.
|
|
12
|
+
class UnsupportedMessageError < Error; end
|
|
13
|
+
|
|
14
|
+
# Raised when a value cannot be decoded for its PostgreSQL OID.
|
|
15
|
+
class ValueDecodeError < Error; end
|
|
16
|
+
|
|
17
|
+
# Raised when DML appears outside an active transaction.
|
|
18
|
+
class TransactionStateError < Error; end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Immutable decoded event objects returned by Pgoutput::Decoder.
|
|
6
|
+
#
|
|
7
|
+
# These Data classes are intentionally value-oriented and are made shareable
|
|
8
|
+
# by Pgoutput::Decoder before they are returned to callers.
|
|
9
|
+
module Events
|
|
10
|
+
# Decoded transaction begin event class.
|
|
11
|
+
#
|
|
12
|
+
# @return [Class]
|
|
13
|
+
Begin = Data.define(:transaction_id, :final_lsn, :commit_timestamp)
|
|
14
|
+
|
|
15
|
+
# Decoded transaction commit event class.
|
|
16
|
+
#
|
|
17
|
+
# @return [Class]
|
|
18
|
+
Commit = Data.define(
|
|
19
|
+
:transaction_id,
|
|
20
|
+
:flags,
|
|
21
|
+
:commit_lsn,
|
|
22
|
+
:transaction_end_lsn,
|
|
23
|
+
:commit_timestamp
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# Decoded insert row-change event class.
|
|
27
|
+
#
|
|
28
|
+
# @return [Class]
|
|
29
|
+
Insert = Data.define(:transaction_id, :relation_id, :schema, :table, :values)
|
|
30
|
+
|
|
31
|
+
# Decoded update row-change event class.
|
|
32
|
+
#
|
|
33
|
+
# @return [Class]
|
|
34
|
+
Update = Data.define(
|
|
35
|
+
:transaction_id,
|
|
36
|
+
:relation_id,
|
|
37
|
+
:schema,
|
|
38
|
+
:table,
|
|
39
|
+
:old_key,
|
|
40
|
+
:old_values,
|
|
41
|
+
:new_values
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
# Decoded delete row-change event class.
|
|
45
|
+
#
|
|
46
|
+
# @return [Class]
|
|
47
|
+
Delete = Data.define(:transaction_id, :relation_id, :schema, :table, :old_key, :old_values)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Mutable per-stream cache of pgoutput-parser Relation messages.
|
|
6
|
+
#
|
|
7
|
+
# @api public
|
|
8
|
+
class RelationCache
|
|
9
|
+
# @return [void]
|
|
10
|
+
def initialize
|
|
11
|
+
@relations = {} # : Hash[Integer, untyped]
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Store a relation message.
|
|
15
|
+
#
|
|
16
|
+
# @param relation [Pgoutput::Messages::Relation]
|
|
17
|
+
# @return [Pgoutput::Messages::Relation]
|
|
18
|
+
def store(relation)
|
|
19
|
+
@relations[relation.relation_id] = relation
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Fetch a relation message by id.
|
|
23
|
+
#
|
|
24
|
+
# @param relation_id [Integer]
|
|
25
|
+
# @return [Pgoutput::Messages::Relation]
|
|
26
|
+
# @raise [UnknownRelationError]
|
|
27
|
+
def fetch(relation_id)
|
|
28
|
+
@relations.fetch(relation_id) do
|
|
29
|
+
raise UnknownRelationError, "unknown relation id #{relation_id}; decode Relation message first"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Builds decoded row hashes from Relation metadata and TupleValue arrays.
|
|
6
|
+
#
|
|
7
|
+
# @api public
|
|
8
|
+
class RowBuilder
|
|
9
|
+
# @param type_registry [TypeRegistry]
|
|
10
|
+
# @return [void]
|
|
11
|
+
def initialize(type_registry: TypeRegistry.default)
|
|
12
|
+
@value_decoder = ValueDecoder.new(type_registry: type_registry)
|
|
13
|
+
freeze
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Build a decoded row hash.
|
|
17
|
+
#
|
|
18
|
+
# @param relation [Pgoutput::Messages::Relation]
|
|
19
|
+
# @param tuple [Array<Pgoutput::Messages::TupleValue>]
|
|
20
|
+
# @return [Hash<String, Object>]
|
|
21
|
+
def build(relation, tuple)
|
|
22
|
+
row = {} # : Hash[String, untyped]
|
|
23
|
+
|
|
24
|
+
tuple.each_with_index do |tuple_value, index|
|
|
25
|
+
column = relation.columns[index]
|
|
26
|
+
next unless column
|
|
27
|
+
|
|
28
|
+
normalized_value = normalize_oid(tuple_value, column.oid)
|
|
29
|
+
row[column.name] = @value_decoder.decode(normalized_value)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
Ractor.make_shareable(row.freeze)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def normalize_oid(tuple_value, oid)
|
|
38
|
+
return tuple_value unless tuple_value.oid.nil?
|
|
39
|
+
|
|
40
|
+
Pgoutput::Messages::TupleValue.new(tuple_value.format, tuple_value.raw, oid)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Immutable PostgreSQL OID-to-decoder registry.
|
|
6
|
+
#
|
|
7
|
+
# The registry maps PostgreSQL type OIDs to callable decoders. It is
|
|
8
|
+
# intentionally separate from pgoutput-parser so the parser remains a pure
|
|
9
|
+
# protocol layer and the decoder owns value conversion policy.
|
|
10
|
+
#
|
|
11
|
+
# Registry instances are immutable after construction. Decoded values are
|
|
12
|
+
# passed through Ractor.make_shareable so caller-visible values can cross
|
|
13
|
+
# Ractor boundaries safely when Ruby supports the value shape.
|
|
14
|
+
#
|
|
15
|
+
# @api public
|
|
16
|
+
class TypeRegistry
|
|
17
|
+
# PostgreSQL bool OID.
|
|
18
|
+
BOOL = 16
|
|
19
|
+
|
|
20
|
+
# PostgreSQL int8 / bigint OID.
|
|
21
|
+
INT8 = 20
|
|
22
|
+
|
|
23
|
+
# PostgreSQL int2 / smallint OID.
|
|
24
|
+
INT2 = 21
|
|
25
|
+
|
|
26
|
+
# PostgreSQL int4 / integer OID.
|
|
27
|
+
INT4 = 23
|
|
28
|
+
|
|
29
|
+
# PostgreSQL text OID.
|
|
30
|
+
TEXT = 25
|
|
31
|
+
|
|
32
|
+
# PostgreSQL json OID.
|
|
33
|
+
JSON = 114
|
|
34
|
+
|
|
35
|
+
# PostgreSQL float4 / real OID.
|
|
36
|
+
FLOAT4 = 700
|
|
37
|
+
|
|
38
|
+
# PostgreSQL float8 / double precision OID.
|
|
39
|
+
FLOAT8 = 701
|
|
40
|
+
|
|
41
|
+
# PostgreSQL varchar OID.
|
|
42
|
+
VARCHAR = 1043
|
|
43
|
+
|
|
44
|
+
# PostgreSQL date OID.
|
|
45
|
+
DATE = 1082
|
|
46
|
+
|
|
47
|
+
# PostgreSQL timestamp without time zone OID.
|
|
48
|
+
TIMESTAMP = 1114
|
|
49
|
+
|
|
50
|
+
# PostgreSQL timestamp with time zone OID.
|
|
51
|
+
TIMESTAMPTZ = 1184
|
|
52
|
+
|
|
53
|
+
# PostgreSQL numeric OID.
|
|
54
|
+
NUMERIC = 1700
|
|
55
|
+
|
|
56
|
+
# PostgreSQL uuid OID.
|
|
57
|
+
UUID = 2950
|
|
58
|
+
|
|
59
|
+
# PostgreSQL jsonb OID.
|
|
60
|
+
JSONB = 3802
|
|
61
|
+
|
|
62
|
+
# Return the process-local default immutable registry.
|
|
63
|
+
#
|
|
64
|
+
# @return [TypeRegistry] default immutable registry.
|
|
65
|
+
def self.default
|
|
66
|
+
@default ||= new(default_decoders)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Build the default OID decoder table.
|
|
70
|
+
#
|
|
71
|
+
# @return [Hash<Integer, Proc>] default decoder table.
|
|
72
|
+
def self.default_decoders
|
|
73
|
+
{
|
|
74
|
+
BOOL => ->(raw, format) { decode_bool(raw, format) },
|
|
75
|
+
INT2 => ->(raw, format) { decode_int(raw, format, 2, "s>") },
|
|
76
|
+
INT4 => ->(raw, format) { decode_int(raw, format, 4, "l>") },
|
|
77
|
+
INT8 => ->(raw, format) { decode_int(raw, format, 8, "q>") },
|
|
78
|
+
TEXT => ->(raw, _format) { raw.dup.freeze },
|
|
79
|
+
VARCHAR => ->(raw, _format) { raw.dup.freeze },
|
|
80
|
+
FLOAT4 => ->(raw, format) { decode_float(raw, format, 4, "g") },
|
|
81
|
+
FLOAT8 => ->(raw, format) { decode_float(raw, format, 8, "G") },
|
|
82
|
+
NUMERIC => ->(raw, format) { format == :text ? Kernel.BigDecimal(raw) : raw.dup.freeze },
|
|
83
|
+
JSON => ->(raw, format) { format == :text ? ::JSON.parse(raw) : raw.dup.freeze },
|
|
84
|
+
JSONB => ->(raw, format) { decode_jsonb(raw, format) },
|
|
85
|
+
UUID => ->(raw, format) { format == :text ? raw.dup.freeze : decode_uuid_binary(raw) },
|
|
86
|
+
DATE => ->(raw, format) { format == :text ? Date.iso8601(raw) : raw.dup.freeze },
|
|
87
|
+
TIMESTAMP => ->(raw, format) { format == :text ? Time.parse(raw) : raw.dup.freeze },
|
|
88
|
+
TIMESTAMPTZ => ->(raw, format) { format == :text ? Time.parse(raw) : raw.dup.freeze }
|
|
89
|
+
}.freeze
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Create an immutable registry.
|
|
93
|
+
#
|
|
94
|
+
# @param decoders [Hash<Integer, Proc>] decoder table.
|
|
95
|
+
# @return [void]
|
|
96
|
+
def initialize(decoders = self.class.default_decoders)
|
|
97
|
+
@decoders = decoders.dup.freeze
|
|
98
|
+
freeze
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Decode a raw tuple payload.
|
|
102
|
+
#
|
|
103
|
+
# @param oid [Integer, nil] PostgreSQL type OID.
|
|
104
|
+
# @param raw [String, nil] raw payload.
|
|
105
|
+
# @param format [Symbol] tuple value format.
|
|
106
|
+
# @return [Object, nil]
|
|
107
|
+
def decode(oid, raw, format)
|
|
108
|
+
return nil if raw.nil?
|
|
109
|
+
|
|
110
|
+
decoder = oid ? @decoders[oid] : nil
|
|
111
|
+
decoded = decoder ? decoder.call(raw, format) : raw.dup.freeze
|
|
112
|
+
Ractor.make_shareable(decoded)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Create a new registry with one custom decoder.
|
|
116
|
+
#
|
|
117
|
+
# @param oid [Integer] PostgreSQL type OID.
|
|
118
|
+
# @yieldparam raw [String] raw payload.
|
|
119
|
+
# @yieldparam format [Symbol] tuple value format.
|
|
120
|
+
# @yieldreturn [Object]
|
|
121
|
+
# @return [TypeRegistry]
|
|
122
|
+
# @raise [ArgumentError] if no block is provided.
|
|
123
|
+
def with_decoder(oid, &block)
|
|
124
|
+
raise ArgumentError, "block required" unless block
|
|
125
|
+
|
|
126
|
+
self.class.new(@decoders.merge(Integer(oid) => block))
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
class << self
|
|
130
|
+
private
|
|
131
|
+
|
|
132
|
+
def decode_bool(raw, format)
|
|
133
|
+
return raw == "t" if format == :text
|
|
134
|
+
|
|
135
|
+
raw.unpack1("C") == 1
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def decode_int(raw, format, expected_length, template)
|
|
139
|
+
return raw.to_i if format == :text
|
|
140
|
+
return raw.dup.freeze unless raw.bytesize == expected_length
|
|
141
|
+
|
|
142
|
+
Integer(raw.unpack1(template))
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def decode_float(raw, format, expected_length, template)
|
|
146
|
+
return Float(raw) if format == :text
|
|
147
|
+
return raw.dup.freeze unless raw.bytesize == expected_length
|
|
148
|
+
|
|
149
|
+
Float(raw.unpack1(template))
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def decode_jsonb(raw, format)
|
|
153
|
+
return ::JSON.parse(raw) if format == :text
|
|
154
|
+
|
|
155
|
+
# PostgreSQL binary jsonb starts with a version byte. Version 1 is the
|
|
156
|
+
# current on-wire format; the remaining bytes contain JSON text.
|
|
157
|
+
return raw.dup.freeze unless raw.bytesize >= 2 && raw.getbyte(0) == 1
|
|
158
|
+
|
|
159
|
+
::JSON.parse(raw.byteslice(1..).to_s)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def decode_uuid_binary(raw)
|
|
163
|
+
return raw.dup.freeze unless raw.bytesize == 16
|
|
164
|
+
|
|
165
|
+
hex = raw.unpack1("H*").to_s
|
|
166
|
+
"#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}".freeze
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Decodes one pgoutput-parser TupleValue using a TypeRegistry.
|
|
6
|
+
#
|
|
7
|
+
# @api public
|
|
8
|
+
class ValueDecoder
|
|
9
|
+
# @param type_registry [TypeRegistry]
|
|
10
|
+
# @return [void]
|
|
11
|
+
def initialize(type_registry: TypeRegistry.default)
|
|
12
|
+
@type_registry = type_registry
|
|
13
|
+
freeze
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Decode one tuple value.
|
|
17
|
+
#
|
|
18
|
+
# @param tuple_value [Pgoutput::Messages::TupleValue]
|
|
19
|
+
# @return [Object, nil, Symbol]
|
|
20
|
+
def decode(tuple_value)
|
|
21
|
+
case tuple_value.format
|
|
22
|
+
when :null
|
|
23
|
+
nil
|
|
24
|
+
when :unchanged_toast
|
|
25
|
+
:unchanged_toast
|
|
26
|
+
when :text, :binary
|
|
27
|
+
@type_registry.decode(tuple_value.oid, tuple_value.raw, tuple_value.format)
|
|
28
|
+
else
|
|
29
|
+
raise ValueDecodeError, "unsupported tuple value format: #{tuple_value.format.inspect}"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
data/lib/pgoutput/decoder.rb
CHANGED
|
@@ -1,10 +1,174 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
require "date"
|
|
5
|
+
require "json"
|
|
6
|
+
require "time"
|
|
7
|
+
require "pgoutput"
|
|
8
|
+
|
|
3
9
|
require_relative "decoder/version"
|
|
10
|
+
require_relative "decoder/errors"
|
|
11
|
+
require_relative "decoder/events"
|
|
12
|
+
require_relative "decoder/type_registry"
|
|
13
|
+
require_relative "decoder/value_decoder"
|
|
14
|
+
require_relative "decoder/relation_cache"
|
|
15
|
+
require_relative "decoder/row_builder"
|
|
4
16
|
|
|
5
17
|
module Pgoutput
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
18
|
+
# Stateful high-level decoder for pgoutput-parser protocol messages.
|
|
19
|
+
#
|
|
20
|
+
# Decoder accepts immutable protocol messages from pgoutput-parser and returns
|
|
21
|
+
# immutable, Ractor-shareable row-change events. The decoder maintains relation
|
|
22
|
+
# and active transaction context, so one instance should be used per logical
|
|
23
|
+
# replication stream.
|
|
24
|
+
#
|
|
25
|
+
# @api public
|
|
26
|
+
class Decoder
|
|
27
|
+
# @return [TypeRegistry]
|
|
28
|
+
attr_reader :type_registry
|
|
29
|
+
|
|
30
|
+
# @param type_registry [TypeRegistry] immutable OID decoder registry.
|
|
31
|
+
# @return [void]
|
|
32
|
+
def initialize(type_registry: TypeRegistry.default)
|
|
33
|
+
@type_registry = type_registry
|
|
34
|
+
@relations = RelationCache.new
|
|
35
|
+
@row_builder = RowBuilder.new(type_registry: type_registry)
|
|
36
|
+
@current_transaction_id = nil
|
|
37
|
+
@current_final_lsn = nil
|
|
38
|
+
@current_commit_timestamp = nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Decode one pgoutput-parser protocol message.
|
|
42
|
+
#
|
|
43
|
+
# @param message [Object] protocol message from pgoutput-parser.
|
|
44
|
+
# @return [Events::Begin, Events::Commit, Events::Insert, Events::Update, Events::Delete, nil]
|
|
45
|
+
# @raise [UnknownRelationError] if a DML message references an unknown relation.
|
|
46
|
+
# @raise [TransactionStateError] if DML arrives before Begin.
|
|
47
|
+
def decode(message)
|
|
48
|
+
case message
|
|
49
|
+
when parser_messages::Begin
|
|
50
|
+
decode_begin(message)
|
|
51
|
+
when parser_messages::Relation
|
|
52
|
+
@relations.store(message)
|
|
53
|
+
nil
|
|
54
|
+
when parser_messages::Insert
|
|
55
|
+
decode_insert(message)
|
|
56
|
+
when parser_messages::Update
|
|
57
|
+
decode_update(message)
|
|
58
|
+
when parser_messages::Delete
|
|
59
|
+
decode_delete(message)
|
|
60
|
+
when parser_messages::Commit
|
|
61
|
+
decode_commit(message)
|
|
62
|
+
else
|
|
63
|
+
raise UnsupportedMessageError, "unsupported message: #{message.class}"
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def decode_begin(message)
|
|
70
|
+
@current_transaction_id = message.xid
|
|
71
|
+
@current_final_lsn = message.final_lsn
|
|
72
|
+
@current_commit_timestamp = message.commit_timestamp
|
|
73
|
+
|
|
74
|
+
share(
|
|
75
|
+
Events::Begin.new(
|
|
76
|
+
message.xid,
|
|
77
|
+
message.final_lsn,
|
|
78
|
+
message.commit_timestamp
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def decode_commit(message)
|
|
84
|
+
transaction_id = @current_transaction_id
|
|
85
|
+
|
|
86
|
+
event = Events::Commit.new(
|
|
87
|
+
transaction_id,
|
|
88
|
+
message.flags,
|
|
89
|
+
message.commit_lsn,
|
|
90
|
+
message.transaction_end_lsn,
|
|
91
|
+
message.commit_timestamp
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
clear_transaction!
|
|
95
|
+
share(event)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def decode_insert(message)
|
|
99
|
+
relation = relation_for(message.relation_id)
|
|
100
|
+
transaction_id = require_transaction_id
|
|
101
|
+
|
|
102
|
+
share(
|
|
103
|
+
Events::Insert.new(
|
|
104
|
+
transaction_id,
|
|
105
|
+
message.relation_id,
|
|
106
|
+
relation.schema,
|
|
107
|
+
relation.table,
|
|
108
|
+
@row_builder.build(relation, message.tuple)
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def decode_update(message)
|
|
114
|
+
relation = relation_for(message.relation_id)
|
|
115
|
+
transaction_id = require_transaction_id
|
|
116
|
+
|
|
117
|
+
share(
|
|
118
|
+
Events::Update.new(
|
|
119
|
+
transaction_id,
|
|
120
|
+
message.relation_id,
|
|
121
|
+
relation.schema,
|
|
122
|
+
relation.table,
|
|
123
|
+
optional_row(relation, message.old_key_tuple),
|
|
124
|
+
optional_row(relation, message.old_tuple),
|
|
125
|
+
@row_builder.build(relation, message.new_tuple)
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def decode_delete(message)
|
|
131
|
+
relation = relation_for(message.relation_id)
|
|
132
|
+
transaction_id = require_transaction_id
|
|
133
|
+
|
|
134
|
+
share(
|
|
135
|
+
Events::Delete.new(
|
|
136
|
+
transaction_id,
|
|
137
|
+
message.relation_id,
|
|
138
|
+
relation.schema,
|
|
139
|
+
relation.table,
|
|
140
|
+
optional_row(relation, message.old_key_tuple),
|
|
141
|
+
optional_row(relation, message.old_tuple)
|
|
142
|
+
)
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def optional_row(relation, tuple)
|
|
147
|
+
return nil if tuple.nil?
|
|
148
|
+
|
|
149
|
+
@row_builder.build(relation, tuple)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def relation_for(relation_id)
|
|
153
|
+
@relations.fetch(relation_id)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def require_transaction_id
|
|
157
|
+
@current_transaction_id || raise(TransactionStateError, "DML message received outside an active transaction")
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def clear_transaction!
|
|
161
|
+
@current_transaction_id = nil
|
|
162
|
+
@current_final_lsn = nil
|
|
163
|
+
@current_commit_timestamp = nil
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def parser_messages
|
|
167
|
+
Pgoutput::Messages
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def share(object)
|
|
171
|
+
Ractor.make_shareable(object)
|
|
172
|
+
end
|
|
9
173
|
end
|
|
10
174
|
end
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
module Pgoutput
|
|
2
|
+
class Decoder
|
|
3
|
+
VERSION: String
|
|
4
|
+
|
|
5
|
+
type parser_message = untyped
|
|
6
|
+
type relation = untyped
|
|
7
|
+
type tuple = Array[untyped]
|
|
8
|
+
type event =
|
|
9
|
+
Events::Begin |
|
|
10
|
+
Events::Commit |
|
|
11
|
+
Events::Insert |
|
|
12
|
+
Events::Update |
|
|
13
|
+
Events::Delete
|
|
14
|
+
|
|
15
|
+
@type_registry: TypeRegistry
|
|
16
|
+
@relations: RelationCache
|
|
17
|
+
@row_builder: RowBuilder
|
|
18
|
+
@current_transaction_id: Integer?
|
|
19
|
+
@current_final_lsn: Integer?
|
|
20
|
+
@current_commit_timestamp: Integer?
|
|
21
|
+
|
|
22
|
+
def initialize: (?type_registry: TypeRegistry) -> void
|
|
23
|
+
def decode: (parser_message message) -> event?
|
|
24
|
+
def type_registry: () -> TypeRegistry
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def decode_begin: (parser_message message) -> Events::Begin
|
|
29
|
+
def decode_commit: (parser_message message) -> Events::Commit
|
|
30
|
+
def decode_insert: (parser_message message) -> Events::Insert
|
|
31
|
+
def decode_update: (parser_message message) -> Events::Update
|
|
32
|
+
def decode_delete: (parser_message message) -> Events::Delete
|
|
33
|
+
def optional_row: (relation relation, tuple? tuple) -> Hash[String, untyped]?
|
|
34
|
+
def relation_for: (Integer relation_id) -> relation
|
|
35
|
+
def require_transaction_id: () -> Integer
|
|
36
|
+
def clear_transaction!: () -> nil
|
|
37
|
+
def parser_messages: () -> untyped
|
|
38
|
+
def share: [T] (T object) -> T
|
|
39
|
+
|
|
40
|
+
class Error < StandardError
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
class UnknownRelationError < Error
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
class UnsupportedMessageError < Error
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
class ValueDecodeError < Error
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
class TransactionStateError < Error
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
module Events
|
|
56
|
+
class Begin < Data
|
|
57
|
+
attr_reader transaction_id: Integer
|
|
58
|
+
attr_reader final_lsn: Integer
|
|
59
|
+
attr_reader commit_timestamp: Integer
|
|
60
|
+
def self.new: (Integer transaction_id, Integer final_lsn, Integer commit_timestamp) -> Begin
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
class Commit < Data
|
|
64
|
+
attr_reader transaction_id: Integer?
|
|
65
|
+
attr_reader flags: Integer
|
|
66
|
+
attr_reader commit_lsn: Integer
|
|
67
|
+
attr_reader transaction_end_lsn: Integer
|
|
68
|
+
attr_reader commit_timestamp: Integer
|
|
69
|
+
def self.new: (Integer? transaction_id, Integer flags, Integer commit_lsn, Integer transaction_end_lsn, Integer commit_timestamp) -> Commit
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
class Insert < Data
|
|
73
|
+
attr_reader transaction_id: Integer
|
|
74
|
+
attr_reader relation_id: Integer
|
|
75
|
+
attr_reader schema: String
|
|
76
|
+
attr_reader table: String
|
|
77
|
+
attr_reader values: Hash[String, untyped]
|
|
78
|
+
def self.new: (Integer transaction_id, Integer relation_id, String schema, String table, Hash[String, untyped] values) -> Insert
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
class Update < Data
|
|
82
|
+
attr_reader transaction_id: Integer
|
|
83
|
+
attr_reader relation_id: Integer
|
|
84
|
+
attr_reader schema: String
|
|
85
|
+
attr_reader table: String
|
|
86
|
+
attr_reader old_key: Hash[String, untyped]?
|
|
87
|
+
attr_reader old_values: Hash[String, untyped]?
|
|
88
|
+
attr_reader new_values: Hash[String, untyped]
|
|
89
|
+
def self.new: (Integer transaction_id, Integer relation_id, String schema, String table, Hash[String, untyped]? old_key, Hash[String, untyped]? old_values, Hash[String, untyped] new_values) -> Update
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
class Delete < Data
|
|
93
|
+
attr_reader transaction_id: Integer
|
|
94
|
+
attr_reader relation_id: Integer
|
|
95
|
+
attr_reader schema: String
|
|
96
|
+
attr_reader table: String
|
|
97
|
+
attr_reader old_key: Hash[String, untyped]?
|
|
98
|
+
attr_reader old_values: Hash[String, untyped]?
|
|
99
|
+
def self.new: (Integer transaction_id, Integer relation_id, String schema, String table, Hash[String, untyped]? old_key, Hash[String, untyped]? old_values) -> Delete
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
class TypeRegistry
|
|
104
|
+
BOOL: Integer
|
|
105
|
+
INT8: Integer
|
|
106
|
+
INT2: Integer
|
|
107
|
+
INT4: Integer
|
|
108
|
+
TEXT: Integer
|
|
109
|
+
JSON: Integer
|
|
110
|
+
FLOAT4: Integer
|
|
111
|
+
FLOAT8: Integer
|
|
112
|
+
VARCHAR: Integer
|
|
113
|
+
DATE: Integer
|
|
114
|
+
TIMESTAMP: Integer
|
|
115
|
+
TIMESTAMPTZ: Integer
|
|
116
|
+
NUMERIC: Integer
|
|
117
|
+
UUID: Integer
|
|
118
|
+
JSONB: Integer
|
|
119
|
+
|
|
120
|
+
type decoder = ^(String raw, Symbol format) -> untyped
|
|
121
|
+
|
|
122
|
+
@decoders: Hash[Integer, decoder]
|
|
123
|
+
|
|
124
|
+
def self.default: () -> TypeRegistry
|
|
125
|
+
def self.default_decoders: () -> Hash[Integer, decoder]
|
|
126
|
+
def initialize: (?Hash[Integer, decoder] decoders) -> void
|
|
127
|
+
def decode: (Integer? oid, String? raw, Symbol format) -> untyped?
|
|
128
|
+
def with_decoder: (Integer oid) { (String raw, Symbol format) -> untyped } -> TypeRegistry
|
|
129
|
+
|
|
130
|
+
private
|
|
131
|
+
|
|
132
|
+
def self.decode_bool: (String raw, Symbol format) -> bool
|
|
133
|
+
def self.decode_int: (String raw, Symbol format, Integer expected_length, String template) -> (Integer | String)
|
|
134
|
+
def self.decode_float: (String raw, Symbol format, Integer expected_length, String template) -> (Float | String)
|
|
135
|
+
def self.decode_jsonb: (String raw, Symbol format) -> untyped
|
|
136
|
+
def self.decode_uuid_binary: (String raw) -> String
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
class ValueDecoder
|
|
140
|
+
@type_registry: TypeRegistry
|
|
141
|
+
|
|
142
|
+
def initialize: (?type_registry: TypeRegistry) -> void
|
|
143
|
+
def decode: (untyped tuple_value) -> untyped?
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
class RelationCache
|
|
147
|
+
@relations: Hash[Integer, relation]
|
|
148
|
+
|
|
149
|
+
def initialize: () -> void
|
|
150
|
+
def store: (relation relation) -> relation
|
|
151
|
+
def fetch: (Integer relation_id) -> relation
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
class RowBuilder
|
|
155
|
+
@value_decoder: ValueDecoder
|
|
156
|
+
|
|
157
|
+
def initialize: (?type_registry: TypeRegistry) -> void
|
|
158
|
+
def build: (relation relation, tuple tuple) -> Hash[String, untyped]
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
def normalize_oid: (untyped tuple_value, Integer oid) -> untyped
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pgoutput-decoder
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.1.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ken C. Demanawa
|
|
@@ -10,117 +10,33 @@ cert_chain: []
|
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
|
-
name:
|
|
13
|
+
name: bigdecimal
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
15
15
|
requirements:
|
|
16
16
|
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '
|
|
18
|
+
version: '4.1'
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - "~>"
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version: '
|
|
26
|
-
- !ruby/object:Gem::Dependency
|
|
27
|
-
name: pry
|
|
28
|
-
requirement: !ruby/object:Gem::Requirement
|
|
29
|
-
requirements:
|
|
30
|
-
- - "~>"
|
|
31
|
-
- !ruby/object:Gem::Version
|
|
32
|
-
version: 0.16.0
|
|
33
|
-
type: :development
|
|
34
|
-
prerelease: false
|
|
35
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
-
requirements:
|
|
37
|
-
- - "~>"
|
|
38
|
-
- !ruby/object:Gem::Version
|
|
39
|
-
version: 0.16.0
|
|
40
|
-
- !ruby/object:Gem::Dependency
|
|
41
|
-
name: minitest
|
|
42
|
-
requirement: !ruby/object:Gem::Requirement
|
|
43
|
-
requirements:
|
|
44
|
-
- - "~>"
|
|
45
|
-
- !ruby/object:Gem::Version
|
|
46
|
-
version: '5.27'
|
|
47
|
-
type: :development
|
|
48
|
-
prerelease: false
|
|
49
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
-
requirements:
|
|
51
|
-
- - "~>"
|
|
52
|
-
- !ruby/object:Gem::Version
|
|
53
|
-
version: '5.27'
|
|
25
|
+
version: '4.1'
|
|
54
26
|
- !ruby/object:Gem::Dependency
|
|
55
|
-
name:
|
|
56
|
-
requirement: !ruby/object:Gem::Requirement
|
|
57
|
-
requirements:
|
|
58
|
-
- - "~>"
|
|
59
|
-
- !ruby/object:Gem::Version
|
|
60
|
-
version: '13.4'
|
|
61
|
-
type: :development
|
|
62
|
-
prerelease: false
|
|
63
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
64
|
-
requirements:
|
|
65
|
-
- - "~>"
|
|
66
|
-
- !ruby/object:Gem::Version
|
|
67
|
-
version: '13.4'
|
|
68
|
-
- !ruby/object:Gem::Dependency
|
|
69
|
-
name: rubocop
|
|
70
|
-
requirement: !ruby/object:Gem::Requirement
|
|
71
|
-
requirements:
|
|
72
|
-
- - "~>"
|
|
73
|
-
- !ruby/object:Gem::Version
|
|
74
|
-
version: '1.87'
|
|
75
|
-
type: :development
|
|
76
|
-
prerelease: false
|
|
77
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
-
requirements:
|
|
79
|
-
- - "~>"
|
|
80
|
-
- !ruby/object:Gem::Version
|
|
81
|
-
version: '1.87'
|
|
82
|
-
- !ruby/object:Gem::Dependency
|
|
83
|
-
name: simplecov
|
|
84
|
-
requirement: !ruby/object:Gem::Requirement
|
|
85
|
-
requirements:
|
|
86
|
-
- - "~>"
|
|
87
|
-
- !ruby/object:Gem::Version
|
|
88
|
-
version: 0.22.0
|
|
89
|
-
type: :development
|
|
90
|
-
prerelease: false
|
|
91
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
92
|
-
requirements:
|
|
93
|
-
- - "~>"
|
|
94
|
-
- !ruby/object:Gem::Version
|
|
95
|
-
version: 0.22.0
|
|
96
|
-
- !ruby/object:Gem::Dependency
|
|
97
|
-
name: steep
|
|
98
|
-
requirement: !ruby/object:Gem::Requirement
|
|
99
|
-
requirements:
|
|
100
|
-
- - "~>"
|
|
101
|
-
- !ruby/object:Gem::Version
|
|
102
|
-
version: '1.10'
|
|
103
|
-
type: :development
|
|
104
|
-
prerelease: false
|
|
105
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
106
|
-
requirements:
|
|
107
|
-
- - "~>"
|
|
108
|
-
- !ruby/object:Gem::Version
|
|
109
|
-
version: '1.10'
|
|
110
|
-
- !ruby/object:Gem::Dependency
|
|
111
|
-
name: yard
|
|
27
|
+
name: pgoutput-parser
|
|
112
28
|
requirement: !ruby/object:Gem::Requirement
|
|
113
29
|
requirements:
|
|
114
30
|
- - "~>"
|
|
115
31
|
- !ruby/object:Gem::Version
|
|
116
|
-
version: 0.
|
|
117
|
-
type: :
|
|
32
|
+
version: '0.1'
|
|
33
|
+
type: :runtime
|
|
118
34
|
prerelease: false
|
|
119
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
120
36
|
requirements:
|
|
121
37
|
- - "~>"
|
|
122
38
|
- !ruby/object:Gem::Version
|
|
123
|
-
version: 0.
|
|
39
|
+
version: '0.1'
|
|
124
40
|
description: Decodes pgoutput-parser protocol messages into immutable Ruby row-change
|
|
125
41
|
events.
|
|
126
42
|
email:
|
|
@@ -133,8 +49,15 @@ files:
|
|
|
133
49
|
- LICENSE.txt
|
|
134
50
|
- README.md
|
|
135
51
|
- lib/pgoutput/decoder.rb
|
|
52
|
+
- lib/pgoutput/decoder/errors.rb
|
|
53
|
+
- lib/pgoutput/decoder/events.rb
|
|
54
|
+
- lib/pgoutput/decoder/relation_cache.rb
|
|
55
|
+
- lib/pgoutput/decoder/row_builder.rb
|
|
56
|
+
- lib/pgoutput/decoder/type_registry.rb
|
|
57
|
+
- lib/pgoutput/decoder/value_decoder.rb
|
|
136
58
|
- lib/pgoutput/decoder/version.rb
|
|
137
|
-
-
|
|
59
|
+
- lib/pgoutput_decoder.rb
|
|
60
|
+
- sig/pgoutput_decoder.rbs
|
|
138
61
|
homepage: https://github.com/kanutocd/pgoutput-decoder
|
|
139
62
|
licenses:
|
|
140
63
|
- MIT
|