pgoutput-decoder 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -2
- data/lib/pgoutput/decoder/errors.rb +20 -0
- data/lib/pgoutput/decoder/events.rb +50 -0
- data/lib/pgoutput/decoder/relation_cache.rb +34 -0
- data/lib/pgoutput/decoder/row_builder.rb +44 -0
- data/lib/pgoutput/decoder/type_registry.rb +171 -0
- data/lib/pgoutput/decoder/value_decoder.rb +34 -0
- data/lib/pgoutput/decoder/version.rb +3 -2
- data/lib/pgoutput/decoder.rb +167 -3
- data/lib/pgoutput_decoder.rb +3 -0
- data/sig/pgoutput_decoder.rbs +119 -0
- metadata +15 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bc6535189e67a471cc685b43e61aa084fd8e5b109cbbff682494a49a936164e6
|
|
4
|
+
data.tar.gz: 93858268fe2519951b33561b5deec1da508f32b7d2fa00f1c364731d5c6715b7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1fbac46c86f50fa10ccdb623cd301ce12b12f72bd271915d0fa567e4605f758b2961cfb466661c1e59918ef89829a8bf367d3e771bcd9d55ec31dbf3d6a2bea3
|
|
7
|
+
data.tar.gz: 3978a4c40f0939917ca97f129130995682602cadf2254b92bee9fa1fe56608baab401112efefc756ed5dc23ce9db837c6702ca9dbb5c71e403db23a1909c5bd1
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
1
8
|
## [Unreleased]
|
|
2
9
|
|
|
3
|
-
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Placeholder for future development.
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## [0.1.0] - 2026-06-01
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- Added initial `pgoutput-decoder` gem structure.
|
|
21
|
+
- Added dependency on `pgoutput-parser`.
|
|
22
|
+
- Added `Pgoutput::Decoder` facade.
|
|
23
|
+
- Added decoded event models for Begin, Commit, Insert, Update, and Delete.
|
|
24
|
+
- Added relation cache for parser Relation messages.
|
|
25
|
+
- Added active transaction tracking using Begin message XID.
|
|
26
|
+
- Added decoded row hash construction from relation columns and tuple values.
|
|
27
|
+
- Added default PostgreSQL OID decoders for common scalar types.
|
|
28
|
+
- Added conservative binary decoding for fixed-width scalar values.
|
|
29
|
+
- Added custom OID decoder support.
|
|
30
|
+
- Added Ractor-shareable decoded event outputs.
|
|
31
|
+
- Added Minitest coverage for registry, value decoding, row building, relation cache, and integration flows.
|
|
32
|
+
- Added RBS signatures.
|
|
33
|
+
- Added README documentation.
|
|
34
|
+
- Added CI and release workflow templates.
|
|
35
|
+
|
|
36
|
+
---
|
|
4
37
|
|
|
5
|
-
-
|
|
38
|
+
[Unreleased]: https://github.com/kanutocd/pgoutput-decoder/compare/v0.1.0...HEAD
|
|
39
|
+
[0.1.0]: https://github.com/kanutocd/pgoutput-decoder/releases/tag/v0.1.0
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Base decoder error.
|
|
6
|
+
class Error < StandardError; end
|
|
7
|
+
|
|
8
|
+
# Raised when a DML message references an unknown relation.
|
|
9
|
+
class UnknownRelationError < Error; end
|
|
10
|
+
|
|
11
|
+
# Raised when a message type cannot be decoded.
|
|
12
|
+
class UnsupportedMessageError < Error; end
|
|
13
|
+
|
|
14
|
+
# Raised when a value cannot be decoded for its PostgreSQL OID.
|
|
15
|
+
class ValueDecodeError < Error; end
|
|
16
|
+
|
|
17
|
+
# Raised when DML appears outside an active transaction.
|
|
18
|
+
class TransactionStateError < Error; end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Immutable decoded event objects returned by Pgoutput::Decoder.
|
|
6
|
+
#
|
|
7
|
+
# These Data classes are intentionally value-oriented and are made shareable
|
|
8
|
+
# by Pgoutput::Decoder before they are returned to callers.
|
|
9
|
+
module Events
|
|
10
|
+
# Decoded transaction begin event class.
|
|
11
|
+
#
|
|
12
|
+
# @return [Class]
|
|
13
|
+
Begin = Data.define(:transaction_id, :final_lsn, :commit_timestamp)
|
|
14
|
+
|
|
15
|
+
# Decoded transaction commit event class.
|
|
16
|
+
#
|
|
17
|
+
# @return [Class]
|
|
18
|
+
Commit = Data.define(
|
|
19
|
+
:transaction_id,
|
|
20
|
+
:flags,
|
|
21
|
+
:commit_lsn,
|
|
22
|
+
:transaction_end_lsn,
|
|
23
|
+
:commit_timestamp
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# Decoded insert row-change event class.
|
|
27
|
+
#
|
|
28
|
+
# @return [Class]
|
|
29
|
+
Insert = Data.define(:transaction_id, :relation_id, :schema, :table, :values)
|
|
30
|
+
|
|
31
|
+
# Decoded update row-change event class.
|
|
32
|
+
#
|
|
33
|
+
# @return [Class]
|
|
34
|
+
Update = Data.define(
|
|
35
|
+
:transaction_id,
|
|
36
|
+
:relation_id,
|
|
37
|
+
:schema,
|
|
38
|
+
:table,
|
|
39
|
+
:old_key,
|
|
40
|
+
:old_values,
|
|
41
|
+
:new_values
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
# Decoded delete row-change event class.
|
|
45
|
+
#
|
|
46
|
+
# @return [Class]
|
|
47
|
+
Delete = Data.define(:transaction_id, :relation_id, :schema, :table, :old_key, :old_values)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Mutable per-stream cache of pgoutput-parser Relation messages.
|
|
6
|
+
#
|
|
7
|
+
# @api public
|
|
8
|
+
class RelationCache
|
|
9
|
+
# @return [void]
|
|
10
|
+
def initialize
|
|
11
|
+
@relations = {} # : Hash[Integer, untyped]
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Store a relation message.
|
|
15
|
+
#
|
|
16
|
+
# @param relation [Pgoutput::Messages::Relation]
|
|
17
|
+
# @return [Pgoutput::Messages::Relation]
|
|
18
|
+
def store(relation)
|
|
19
|
+
@relations[relation.relation_id] = relation
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Fetch a relation message by id.
|
|
23
|
+
#
|
|
24
|
+
# @param relation_id [Integer]
|
|
25
|
+
# @return [Pgoutput::Messages::Relation]
|
|
26
|
+
# @raise [UnknownRelationError]
|
|
27
|
+
def fetch(relation_id)
|
|
28
|
+
@relations.fetch(relation_id) do
|
|
29
|
+
raise UnknownRelationError, "unknown relation id #{relation_id}; decode Relation message first"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Builds decoded row hashes from Relation metadata and TupleValue arrays.
|
|
6
|
+
#
|
|
7
|
+
# @api public
|
|
8
|
+
class RowBuilder
|
|
9
|
+
# @param type_registry [TypeRegistry]
|
|
10
|
+
# @return [void]
|
|
11
|
+
def initialize(type_registry: TypeRegistry.default)
|
|
12
|
+
@value_decoder = ValueDecoder.new(type_registry: type_registry)
|
|
13
|
+
freeze
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Build a decoded row hash.
|
|
17
|
+
#
|
|
18
|
+
# @param relation [Pgoutput::Messages::Relation]
|
|
19
|
+
# @param tuple [Array<Pgoutput::Messages::TupleValue>]
|
|
20
|
+
# @return [Hash<String, Object>]
|
|
21
|
+
def build(relation, tuple)
|
|
22
|
+
row = {}
|
|
23
|
+
|
|
24
|
+
tuple.each_with_index do |tuple_value, index|
|
|
25
|
+
column = relation.columns[index]
|
|
26
|
+
next unless column
|
|
27
|
+
|
|
28
|
+
normalized_value = normalize_oid(tuple_value, column.oid)
|
|
29
|
+
row[column.name] = @value_decoder.decode(normalized_value)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
Ractor.make_shareable(row.freeze)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def normalize_oid(tuple_value, oid)
|
|
38
|
+
return tuple_value unless tuple_value.oid.nil?
|
|
39
|
+
|
|
40
|
+
Pgoutput::Messages::TupleValue.new(tuple_value.format, tuple_value.raw, oid)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Immutable PostgreSQL OID-to-decoder registry.
|
|
6
|
+
#
|
|
7
|
+
# The registry maps PostgreSQL type OIDs to callable decoders. It is
|
|
8
|
+
# intentionally separate from pgoutput-parser so the parser remains a pure
|
|
9
|
+
# protocol layer and the decoder owns value conversion policy.
|
|
10
|
+
#
|
|
11
|
+
# Registry instances are immutable after construction. Decoded values are
|
|
12
|
+
# passed through Ractor.make_shareable so caller-visible values can cross
|
|
13
|
+
# Ractor boundaries safely when Ruby supports the value shape.
|
|
14
|
+
#
|
|
15
|
+
# @api public
|
|
16
|
+
class TypeRegistry
|
|
17
|
+
# PostgreSQL bool OID.
|
|
18
|
+
BOOL = 16
|
|
19
|
+
|
|
20
|
+
# PostgreSQL int8 / bigint OID.
|
|
21
|
+
INT8 = 20
|
|
22
|
+
|
|
23
|
+
# PostgreSQL int2 / smallint OID.
|
|
24
|
+
INT2 = 21
|
|
25
|
+
|
|
26
|
+
# PostgreSQL int4 / integer OID.
|
|
27
|
+
INT4 = 23
|
|
28
|
+
|
|
29
|
+
# PostgreSQL text OID.
|
|
30
|
+
TEXT = 25
|
|
31
|
+
|
|
32
|
+
# PostgreSQL json OID.
|
|
33
|
+
JSON = 114
|
|
34
|
+
|
|
35
|
+
# PostgreSQL float4 / real OID.
|
|
36
|
+
FLOAT4 = 700
|
|
37
|
+
|
|
38
|
+
# PostgreSQL float8 / double precision OID.
|
|
39
|
+
FLOAT8 = 701
|
|
40
|
+
|
|
41
|
+
# PostgreSQL varchar OID.
|
|
42
|
+
VARCHAR = 1043
|
|
43
|
+
|
|
44
|
+
# PostgreSQL date OID.
|
|
45
|
+
DATE = 1082
|
|
46
|
+
|
|
47
|
+
# PostgreSQL timestamp without time zone OID.
|
|
48
|
+
TIMESTAMP = 1114
|
|
49
|
+
|
|
50
|
+
# PostgreSQL timestamp with time zone OID.
|
|
51
|
+
TIMESTAMPTZ = 1184
|
|
52
|
+
|
|
53
|
+
# PostgreSQL numeric OID.
|
|
54
|
+
NUMERIC = 1700
|
|
55
|
+
|
|
56
|
+
# PostgreSQL uuid OID.
|
|
57
|
+
UUID = 2950
|
|
58
|
+
|
|
59
|
+
# PostgreSQL jsonb OID.
|
|
60
|
+
JSONB = 3802
|
|
61
|
+
|
|
62
|
+
# Return the process-local default immutable registry.
|
|
63
|
+
#
|
|
64
|
+
# @return [TypeRegistry] default immutable registry.
|
|
65
|
+
def self.default
|
|
66
|
+
@default ||= new(default_decoders)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Build the default OID decoder table.
|
|
70
|
+
#
|
|
71
|
+
# @return [Hash<Integer, Proc>] default decoder table.
|
|
72
|
+
def self.default_decoders
|
|
73
|
+
{
|
|
74
|
+
BOOL => ->(raw, format) { decode_bool(raw, format) },
|
|
75
|
+
INT2 => ->(raw, format) { decode_int(raw, format, 2, "s>") },
|
|
76
|
+
INT4 => ->(raw, format) { decode_int(raw, format, 4, "l>") },
|
|
77
|
+
INT8 => ->(raw, format) { decode_int(raw, format, 8, "q>") },
|
|
78
|
+
TEXT => ->(raw, _format) { raw.dup.freeze },
|
|
79
|
+
VARCHAR => ->(raw, _format) { raw.dup.freeze },
|
|
80
|
+
FLOAT4 => ->(raw, format) { decode_float(raw, format, 4, "g") },
|
|
81
|
+
FLOAT8 => ->(raw, format) { decode_float(raw, format, 8, "G") },
|
|
82
|
+
NUMERIC => ->(raw, format) { format == :text ? BigDecimal(raw) : raw.dup.freeze },
|
|
83
|
+
JSON => ->(raw, format) { format == :text ? ::JSON.parse(raw) : raw.dup.freeze },
|
|
84
|
+
JSONB => ->(raw, format) { decode_jsonb(raw, format) },
|
|
85
|
+
UUID => ->(raw, format) { format == :text ? raw.dup.freeze : decode_uuid_binary(raw) },
|
|
86
|
+
DATE => ->(raw, format) { format == :text ? Date.iso8601(raw) : raw.dup.freeze },
|
|
87
|
+
TIMESTAMP => ->(raw, format) { format == :text ? Time.parse(raw) : raw.dup.freeze },
|
|
88
|
+
TIMESTAMPTZ => ->(raw, format) { format == :text ? Time.parse(raw) : raw.dup.freeze }
|
|
89
|
+
}.freeze
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Create an immutable registry.
|
|
93
|
+
#
|
|
94
|
+
# @param decoders [Hash<Integer, Proc>] decoder table.
|
|
95
|
+
# @return [void]
|
|
96
|
+
def initialize(decoders = self.class.default_decoders)
|
|
97
|
+
@decoders = decoders.dup.freeze
|
|
98
|
+
freeze
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Decode a raw tuple payload.
|
|
102
|
+
#
|
|
103
|
+
# @param oid [Integer, nil] PostgreSQL type OID.
|
|
104
|
+
# @param raw [String, nil] raw payload.
|
|
105
|
+
# @param format [Symbol] tuple value format.
|
|
106
|
+
# @return [Object, nil]
|
|
107
|
+
def decode(oid, raw, format)
|
|
108
|
+
return nil if raw.nil?
|
|
109
|
+
|
|
110
|
+
decoder = @decoders[oid]
|
|
111
|
+
decoded = decoder ? decoder.call(raw, format) : raw.dup.freeze
|
|
112
|
+
Ractor.make_shareable(decoded)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Create a new registry with one custom decoder.
|
|
116
|
+
#
|
|
117
|
+
# @param oid [Integer] PostgreSQL type OID.
|
|
118
|
+
# @yieldparam raw [String] raw payload.
|
|
119
|
+
# @yieldparam format [Symbol] tuple value format.
|
|
120
|
+
# @yieldreturn [Object]
|
|
121
|
+
# @return [TypeRegistry]
|
|
122
|
+
# @raise [ArgumentError] if no block is provided.
|
|
123
|
+
def with_decoder(oid, &block)
|
|
124
|
+
raise ArgumentError, "block required" unless block
|
|
125
|
+
|
|
126
|
+
self.class.new(@decoders.merge(Integer(oid) => block))
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
class << self
|
|
130
|
+
private
|
|
131
|
+
|
|
132
|
+
def decode_bool(raw, format)
|
|
133
|
+
return raw == "t" if format == :text
|
|
134
|
+
|
|
135
|
+
raw.unpack1("C") == 1
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def decode_int(raw, format, expected_length, template)
|
|
139
|
+
return raw.to_i if format == :text
|
|
140
|
+
return raw.dup.freeze unless raw.bytesize == expected_length
|
|
141
|
+
|
|
142
|
+
raw.unpack1(template)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def decode_float(raw, format, expected_length, template)
|
|
146
|
+
return Float(raw) if format == :text
|
|
147
|
+
return raw.dup.freeze unless raw.bytesize == expected_length
|
|
148
|
+
|
|
149
|
+
raw.unpack1(template)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def decode_jsonb(raw, format)
|
|
153
|
+
return ::JSON.parse(raw) if format == :text
|
|
154
|
+
|
|
155
|
+
# PostgreSQL binary jsonb starts with a version byte. Version 1 is the
|
|
156
|
+
# current on-wire format; the remaining bytes contain JSON text.
|
|
157
|
+
return raw.dup.freeze unless raw.bytesize >= 2 && raw.getbyte(0) == 1
|
|
158
|
+
|
|
159
|
+
::JSON.parse(raw.byteslice(1..))
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def decode_uuid_binary(raw)
|
|
163
|
+
return raw.dup.freeze unless raw.bytesize == 16
|
|
164
|
+
|
|
165
|
+
hex = raw.unpack1("H*")
|
|
166
|
+
"#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}".freeze
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgoutput
|
|
4
|
+
class Decoder
|
|
5
|
+
# Decodes one pgoutput-parser TupleValue using a TypeRegistry.
|
|
6
|
+
#
|
|
7
|
+
# @api public
|
|
8
|
+
class ValueDecoder
|
|
9
|
+
# @param type_registry [TypeRegistry]
|
|
10
|
+
# @return [void]
|
|
11
|
+
def initialize(type_registry: TypeRegistry.default)
|
|
12
|
+
@type_registry = type_registry
|
|
13
|
+
freeze
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Decode one tuple value.
|
|
17
|
+
#
|
|
18
|
+
# @param tuple_value [Pgoutput::Messages::TupleValue]
|
|
19
|
+
# @return [Object, nil, Symbol]
|
|
20
|
+
def decode(tuple_value)
|
|
21
|
+
case tuple_value.format
|
|
22
|
+
when :null
|
|
23
|
+
nil
|
|
24
|
+
when :unchanged_toast
|
|
25
|
+
:unchanged_toast
|
|
26
|
+
when :text, :binary
|
|
27
|
+
@type_registry.decode(tuple_value.oid, tuple_value.raw, tuple_value.format)
|
|
28
|
+
else
|
|
29
|
+
raise ValueDecodeError, "unsupported tuple value format: #{tuple_value.format.inspect}"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
data/lib/pgoutput/decoder.rb
CHANGED
|
@@ -1,10 +1,174 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
require "date"
|
|
5
|
+
require "json"
|
|
6
|
+
require "time"
|
|
7
|
+
require "pgoutput"
|
|
8
|
+
|
|
3
9
|
require_relative "decoder/version"
|
|
10
|
+
require_relative "decoder/errors"
|
|
11
|
+
require_relative "decoder/events"
|
|
12
|
+
require_relative "decoder/type_registry"
|
|
13
|
+
require_relative "decoder/value_decoder"
|
|
14
|
+
require_relative "decoder/relation_cache"
|
|
15
|
+
require_relative "decoder/row_builder"
|
|
4
16
|
|
|
5
17
|
module Pgoutput
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
18
|
+
# Stateful high-level decoder for pgoutput-parser protocol messages.
|
|
19
|
+
#
|
|
20
|
+
# Decoder accepts immutable protocol messages from pgoutput-parser and returns
|
|
21
|
+
# immutable, Ractor-shareable row-change events. The decoder maintains relation
|
|
22
|
+
# and active transaction context, so one instance should be used per logical
|
|
23
|
+
# replication stream.
|
|
24
|
+
#
|
|
25
|
+
# @api public
|
|
26
|
+
class Decoder
|
|
27
|
+
# @return [TypeRegistry]
|
|
28
|
+
attr_reader :type_registry
|
|
29
|
+
|
|
30
|
+
# @param type_registry [TypeRegistry] immutable OID decoder registry.
|
|
31
|
+
# @return [void]
|
|
32
|
+
def initialize(type_registry: TypeRegistry.default)
|
|
33
|
+
@type_registry = type_registry
|
|
34
|
+
@relations = RelationCache.new
|
|
35
|
+
@row_builder = RowBuilder.new(type_registry: type_registry)
|
|
36
|
+
@current_transaction_id = nil
|
|
37
|
+
@current_final_lsn = nil
|
|
38
|
+
@current_commit_timestamp = nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Decode one pgoutput-parser protocol message.
|
|
42
|
+
#
|
|
43
|
+
# @param message [Object] protocol message from pgoutput-parser.
|
|
44
|
+
# @return [Events::Begin, Events::Commit, Events::Insert, Events::Update, Events::Delete, nil]
|
|
45
|
+
# @raise [UnknownRelationError] if a DML message references an unknown relation.
|
|
46
|
+
# @raise [TransactionStateError] if DML arrives before Begin.
|
|
47
|
+
def decode(message)
|
|
48
|
+
case message
|
|
49
|
+
when parser_messages::Begin
|
|
50
|
+
decode_begin(message)
|
|
51
|
+
when parser_messages::Relation
|
|
52
|
+
@relations.store(message)
|
|
53
|
+
nil
|
|
54
|
+
when parser_messages::Insert
|
|
55
|
+
decode_insert(message)
|
|
56
|
+
when parser_messages::Update
|
|
57
|
+
decode_update(message)
|
|
58
|
+
when parser_messages::Delete
|
|
59
|
+
decode_delete(message)
|
|
60
|
+
when parser_messages::Commit
|
|
61
|
+
decode_commit(message)
|
|
62
|
+
else
|
|
63
|
+
raise UnsupportedMessageError, "unsupported message: #{message.class}"
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def decode_begin(message)
|
|
70
|
+
@current_transaction_id = message.xid
|
|
71
|
+
@current_final_lsn = message.final_lsn
|
|
72
|
+
@current_commit_timestamp = message.commit_timestamp
|
|
73
|
+
|
|
74
|
+
share(
|
|
75
|
+
Events::Begin.new(
|
|
76
|
+
message.xid,
|
|
77
|
+
message.final_lsn,
|
|
78
|
+
message.commit_timestamp
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def decode_commit(message)
|
|
84
|
+
transaction_id = @current_transaction_id
|
|
85
|
+
|
|
86
|
+
event = Events::Commit.new(
|
|
87
|
+
transaction_id,
|
|
88
|
+
message.flags,
|
|
89
|
+
message.commit_lsn,
|
|
90
|
+
message.transaction_end_lsn,
|
|
91
|
+
message.commit_timestamp
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
clear_transaction!
|
|
95
|
+
share(event)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def decode_insert(message)
|
|
99
|
+
relation = relation_for(message.relation_id)
|
|
100
|
+
transaction_id = require_transaction_id
|
|
101
|
+
|
|
102
|
+
share(
|
|
103
|
+
Events::Insert.new(
|
|
104
|
+
transaction_id,
|
|
105
|
+
message.relation_id,
|
|
106
|
+
relation.schema,
|
|
107
|
+
relation.table,
|
|
108
|
+
@row_builder.build(relation, message.tuple)
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def decode_update(message)
|
|
114
|
+
relation = relation_for(message.relation_id)
|
|
115
|
+
transaction_id = require_transaction_id
|
|
116
|
+
|
|
117
|
+
share(
|
|
118
|
+
Events::Update.new(
|
|
119
|
+
transaction_id,
|
|
120
|
+
message.relation_id,
|
|
121
|
+
relation.schema,
|
|
122
|
+
relation.table,
|
|
123
|
+
optional_row(relation, message.old_key_tuple),
|
|
124
|
+
optional_row(relation, message.old_tuple),
|
|
125
|
+
@row_builder.build(relation, message.new_tuple)
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def decode_delete(message)
|
|
131
|
+
relation = relation_for(message.relation_id)
|
|
132
|
+
transaction_id = require_transaction_id
|
|
133
|
+
|
|
134
|
+
share(
|
|
135
|
+
Events::Delete.new(
|
|
136
|
+
transaction_id,
|
|
137
|
+
message.relation_id,
|
|
138
|
+
relation.schema,
|
|
139
|
+
relation.table,
|
|
140
|
+
optional_row(relation, message.old_key_tuple),
|
|
141
|
+
optional_row(relation, message.old_tuple)
|
|
142
|
+
)
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def optional_row(relation, tuple)
|
|
147
|
+
return nil if tuple.nil?
|
|
148
|
+
|
|
149
|
+
@row_builder.build(relation, tuple)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def relation_for(relation_id)
|
|
153
|
+
@relations.fetch(relation_id)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def require_transaction_id
|
|
157
|
+
@current_transaction_id || raise(TransactionStateError, "DML message received outside an active transaction")
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def clear_transaction!
|
|
161
|
+
@current_transaction_id = nil
|
|
162
|
+
@current_final_lsn = nil
|
|
163
|
+
@current_commit_timestamp = nil
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def parser_messages
|
|
167
|
+
Pgoutput::Messages
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def share(object)
|
|
171
|
+
Ractor.make_shareable(object)
|
|
172
|
+
end
|
|
9
173
|
end
|
|
10
174
|
end
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
module Pgoutput
|
|
2
|
+
class Decoder
|
|
3
|
+
VERSION: String
|
|
4
|
+
|
|
5
|
+
type event =
|
|
6
|
+
Events::Begin |
|
|
7
|
+
Events::Commit |
|
|
8
|
+
Events::Insert |
|
|
9
|
+
Events::Update |
|
|
10
|
+
Events::Delete
|
|
11
|
+
|
|
12
|
+
def initialize: (?type_registry: TypeRegistry) -> void
|
|
13
|
+
def decode: (untyped message) -> event?
|
|
14
|
+
def type_registry: () -> TypeRegistry
|
|
15
|
+
|
|
16
|
+
class Error < StandardError
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
class UnknownRelationError < Error
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
class UnsupportedMessageError < Error
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
class ValueDecodeError < Error
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
class TransactionStateError < Error
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
module Events
|
|
32
|
+
class Begin < Data
|
|
33
|
+
attr_reader transaction_id: Integer
|
|
34
|
+
attr_reader final_lsn: Integer
|
|
35
|
+
attr_reader commit_timestamp: Integer
|
|
36
|
+
def self.new: (Integer transaction_id, Integer final_lsn, Integer commit_timestamp) -> Begin
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
class Commit < Data
|
|
40
|
+
attr_reader transaction_id: Integer?
|
|
41
|
+
attr_reader flags: Integer
|
|
42
|
+
attr_reader commit_lsn: Integer
|
|
43
|
+
attr_reader transaction_end_lsn: Integer
|
|
44
|
+
attr_reader commit_timestamp: Integer
|
|
45
|
+
def self.new: (Integer? transaction_id, Integer flags, Integer commit_lsn, Integer transaction_end_lsn, Integer commit_timestamp) -> Commit
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
class Insert < Data
|
|
49
|
+
attr_reader transaction_id: Integer
|
|
50
|
+
attr_reader relation_id: Integer
|
|
51
|
+
attr_reader schema: String
|
|
52
|
+
attr_reader table: String
|
|
53
|
+
attr_reader values: Hash[String, untyped]
|
|
54
|
+
def self.new: (Integer transaction_id, Integer relation_id, String schema, String table, Hash[String, untyped] values) -> Insert
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
class Update < Data
|
|
58
|
+
attr_reader transaction_id: Integer
|
|
59
|
+
attr_reader relation_id: Integer
|
|
60
|
+
attr_reader schema: String
|
|
61
|
+
attr_reader table: String
|
|
62
|
+
attr_reader old_key: Hash[String, untyped]?
|
|
63
|
+
attr_reader old_values: Hash[String, untyped]?
|
|
64
|
+
attr_reader new_values: Hash[String, untyped]
|
|
65
|
+
def self.new: (Integer transaction_id, Integer relation_id, String schema, String table, Hash[String, untyped]? old_key, Hash[String, untyped]? old_values, Hash[String, untyped] new_values) -> Update
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
class Delete < Data
|
|
69
|
+
attr_reader transaction_id: Integer
|
|
70
|
+
attr_reader relation_id: Integer
|
|
71
|
+
attr_reader schema: String
|
|
72
|
+
attr_reader table: String
|
|
73
|
+
attr_reader old_key: Hash[String, untyped]?
|
|
74
|
+
attr_reader old_values: Hash[String, untyped]?
|
|
75
|
+
def self.new: (Integer transaction_id, Integer relation_id, String schema, String table, Hash[String, untyped]? old_key, Hash[String, untyped]? old_values) -> Delete
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
class TypeRegistry
|
|
80
|
+
BOOL: Integer
|
|
81
|
+
INT8: Integer
|
|
82
|
+
INT2: Integer
|
|
83
|
+
INT4: Integer
|
|
84
|
+
TEXT: Integer
|
|
85
|
+
JSON: Integer
|
|
86
|
+
FLOAT4: Integer
|
|
87
|
+
FLOAT8: Integer
|
|
88
|
+
VARCHAR: Integer
|
|
89
|
+
DATE: Integer
|
|
90
|
+
TIMESTAMP: Integer
|
|
91
|
+
TIMESTAMPTZ: Integer
|
|
92
|
+
NUMERIC: Integer
|
|
93
|
+
UUID: Integer
|
|
94
|
+
JSONB: Integer
|
|
95
|
+
|
|
96
|
+
def self.default: () -> TypeRegistry
|
|
97
|
+
def self.default_decoders: () -> Hash[Integer, Proc]
|
|
98
|
+
def initialize: (?Hash[Integer, Proc] decoders) -> void
|
|
99
|
+
def decode: (Integer? oid, String? raw, Symbol format) -> untyped?
|
|
100
|
+
def with_decoder: (Integer oid) { (String raw, Symbol format) -> untyped } -> TypeRegistry
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
class ValueDecoder
|
|
104
|
+
def initialize: (?type_registry: TypeRegistry) -> void
|
|
105
|
+
def decode: (untyped tuple_value) -> untyped?
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
class RelationCache
|
|
109
|
+
def initialize: () -> void
|
|
110
|
+
def store: (untyped relation) -> untyped
|
|
111
|
+
def fetch: (Integer relation_id) -> untyped
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
class RowBuilder
|
|
115
|
+
def initialize: (?type_registry: TypeRegistry) -> void
|
|
116
|
+
def build: (untyped relation, Array[untyped] tuple) -> Hash[String, untyped]
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pgoutput-decoder
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ken C. Demanawa
|
|
@@ -24,33 +24,33 @@ dependencies:
|
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '0.1'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
|
-
name:
|
|
27
|
+
name: minitest
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
30
|
- - "~>"
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version:
|
|
32
|
+
version: '5.27'
|
|
33
33
|
type: :development
|
|
34
34
|
prerelease: false
|
|
35
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version:
|
|
39
|
+
version: '5.27'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
|
-
name:
|
|
41
|
+
name: pry
|
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
|
43
43
|
requirements:
|
|
44
44
|
- - "~>"
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
|
-
version:
|
|
46
|
+
version: 0.16.0
|
|
47
47
|
type: :development
|
|
48
48
|
prerelease: false
|
|
49
49
|
version_requirements: !ruby/object:Gem::Requirement
|
|
50
50
|
requirements:
|
|
51
51
|
- - "~>"
|
|
52
52
|
- !ruby/object:Gem::Version
|
|
53
|
-
version:
|
|
53
|
+
version: 0.16.0
|
|
54
54
|
- !ruby/object:Gem::Dependency
|
|
55
55
|
name: rake
|
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -133,8 +133,16 @@ files:
|
|
|
133
133
|
- LICENSE.txt
|
|
134
134
|
- README.md
|
|
135
135
|
- lib/pgoutput/decoder.rb
|
|
136
|
+
- lib/pgoutput/decoder/errors.rb
|
|
137
|
+
- lib/pgoutput/decoder/events.rb
|
|
138
|
+
- lib/pgoutput/decoder/relation_cache.rb
|
|
139
|
+
- lib/pgoutput/decoder/row_builder.rb
|
|
140
|
+
- lib/pgoutput/decoder/type_registry.rb
|
|
141
|
+
- lib/pgoutput/decoder/value_decoder.rb
|
|
136
142
|
- lib/pgoutput/decoder/version.rb
|
|
143
|
+
- lib/pgoutput_decoder.rb
|
|
137
144
|
- sig/pgoutput/decoder.rbs
|
|
145
|
+
- sig/pgoutput_decoder.rbs
|
|
138
146
|
homepage: https://github.com/kanutocd/pgoutput-decoder
|
|
139
147
|
licenses:
|
|
140
148
|
- MIT
|