mysql_replicator 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +79 -0
- data/CHANGELOG.md +5 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/LICENSE.txt +21 -0
- data/README.md +115 -0
- data/Rakefile +12 -0
- data/Steepfile +22 -0
- data/docker-compose.yml +13 -0
- data/lib/mysql_replicator/binlog_client.rb +201 -0
- data/lib/mysql_replicator/binlogs/column_parser.rb +425 -0
- data/lib/mysql_replicator/binlogs/constants.rb +74 -0
- data/lib/mysql_replicator/binlogs/event_parser.rb +134 -0
- data/lib/mysql_replicator/binlogs/format_description_event_parser.rb +24 -0
- data/lib/mysql_replicator/binlogs/json_parser.rb +335 -0
- data/lib/mysql_replicator/binlogs/query_event_parser.rb +69 -0
- data/lib/mysql_replicator/binlogs/rotate_event_parser.rb +37 -0
- data/lib/mysql_replicator/binlogs/rows_event_parser.rb +161 -0
- data/lib/mysql_replicator/binlogs/table_map_event_parser.rb +155 -0
- data/lib/mysql_replicator/binlogs/xid_event_parser.rb +25 -0
- data/lib/mysql_replicator/connection.rb +226 -0
- data/lib/mysql_replicator/connections/auth.rb +303 -0
- data/lib/mysql_replicator/connections/handshake.rb +132 -0
- data/lib/mysql_replicator/connections/query.rb +322 -0
- data/lib/mysql_replicator/error.rb +6 -0
- data/lib/mysql_replicator/logger.rb +43 -0
- data/lib/mysql_replicator/string_io_util.rb +199 -0
- data/lib/mysql_replicator/string_util.rb +106 -0
- data/lib/mysql_replicator/version.rb +6 -0
- data/lib/mysql_replicator.rb +51 -0
- data/sig/generated/mysql_replicator/binlog_client.rbs +52 -0
- data/sig/generated/mysql_replicator/binlogs/column_parser.rbs +134 -0
- data/sig/generated/mysql_replicator/binlogs/constants.rbs +69 -0
- data/sig/generated/mysql_replicator/binlogs/event_parser.rbs +35 -0
- data/sig/generated/mysql_replicator/binlogs/format_description_event_parser.rbs +13 -0
- data/sig/generated/mysql_replicator/binlogs/json_parser.rbs +101 -0
- data/sig/generated/mysql_replicator/binlogs/query_event_parser.rbs +14 -0
- data/sig/generated/mysql_replicator/binlogs/rotate_event_parser.rbs +14 -0
- data/sig/generated/mysql_replicator/binlogs/rows_event_parser.rbs +39 -0
- data/sig/generated/mysql_replicator/binlogs/table_map_event_parser.rbs +31 -0
- data/sig/generated/mysql_replicator/binlogs/xid_event_parser.rbs +13 -0
- data/sig/generated/mysql_replicator/connection.rbs +103 -0
- data/sig/generated/mysql_replicator/connections/auth.rbs +76 -0
- data/sig/generated/mysql_replicator/connections/handshake.rbs +21 -0
- data/sig/generated/mysql_replicator/connections/query.rbs +62 -0
- data/sig/generated/mysql_replicator/error.rbs +6 -0
- data/sig/generated/mysql_replicator/logger.rbs +26 -0
- data/sig/generated/mysql_replicator/string_io_util.rbs +75 -0
- data/sig/generated/mysql_replicator/string_util.rbs +45 -0
- data/sig/generated/mysql_replicator/types.rbs +19 -0
- data/sig/generated/mysql_replicator/version.rbs +5 -0
- data/sig/generated/mysql_replicator.rbs +16 -0
- metadata +124 -0
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require 'bigdecimal'
|
|
5
|
+
require 'json'
|
|
6
|
+
|
|
7
|
+
module MysqlReplicator
|
|
8
|
+
module Binlogs
|
|
9
|
+
class ColumnParser
|
|
10
|
+
# @rbs io: StringIO
|
|
11
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
12
|
+
def self.parse(io, column_def)
|
|
13
|
+
type_code = MysqlReplicator::Binlogs::FieldTypes.code_for(column_def[:data_type])
|
|
14
|
+
|
|
15
|
+
case type_code
|
|
16
|
+
when MysqlReplicator::Binlogs::FieldTypes::TINY_INT
|
|
17
|
+
parse_tinyint(io)
|
|
18
|
+
when MysqlReplicator::Binlogs::FieldTypes::SMALL_INT
|
|
19
|
+
parse_smallint(io)
|
|
20
|
+
when MysqlReplicator::Binlogs::FieldTypes::MEDIUM_INT
|
|
21
|
+
parse_mediumint(io)
|
|
22
|
+
when MysqlReplicator::Binlogs::FieldTypes::INT
|
|
23
|
+
parse_int(io)
|
|
24
|
+
when MysqlReplicator::Binlogs::FieldTypes::BIG_INT
|
|
25
|
+
parse_bigint(io)
|
|
26
|
+
when MysqlReplicator::Binlogs::FieldTypes::FLOAT
|
|
27
|
+
parse_float(io)
|
|
28
|
+
when MysqlReplicator::Binlogs::FieldTypes::DOUBLE
|
|
29
|
+
parse_double(io)
|
|
30
|
+
when MysqlReplicator::Binlogs::FieldTypes::DECIMAL
|
|
31
|
+
parse_decimal(io, column_def)
|
|
32
|
+
when MysqlReplicator::Binlogs::FieldTypes::DATETIME
|
|
33
|
+
parse_datetime(io)
|
|
34
|
+
when MysqlReplicator::Binlogs::FieldTypes::DATE
|
|
35
|
+
parse_date(io)
|
|
36
|
+
when MysqlReplicator::Binlogs::FieldTypes::TIME
|
|
37
|
+
parse_time(io)
|
|
38
|
+
when MysqlReplicator::Binlogs::FieldTypes::TIMESTAMP
|
|
39
|
+
parse_timestamp(io)
|
|
40
|
+
when MysqlReplicator::Binlogs::FieldTypes::CHAR
|
|
41
|
+
parse_char(io, column_def)
|
|
42
|
+
when MysqlReplicator::Binlogs::FieldTypes::VARCHAR
|
|
43
|
+
parse_varchar(io, column_def)
|
|
44
|
+
when MysqlReplicator::Binlogs::FieldTypes::TINY_TEXT
|
|
45
|
+
parse_tinytext(io, column_def)
|
|
46
|
+
when MysqlReplicator::Binlogs::FieldTypes::TEXT
|
|
47
|
+
parse_text(io, column_def)
|
|
48
|
+
when MysqlReplicator::Binlogs::FieldTypes::MEDIUM_TEXT
|
|
49
|
+
parse_mediumtext(io, column_def)
|
|
50
|
+
when MysqlReplicator::Binlogs::FieldTypes::LONG_TEXT
|
|
51
|
+
parse_longtext(io, column_def)
|
|
52
|
+
when MysqlReplicator::Binlogs::FieldTypes::TINY_BLOB
|
|
53
|
+
parse_tinyblob(io)
|
|
54
|
+
when MysqlReplicator::Binlogs::FieldTypes::BLOB
|
|
55
|
+
parse_blob(io)
|
|
56
|
+
when MysqlReplicator::Binlogs::FieldTypes::MEDIUM_BLOB
|
|
57
|
+
parse_mediumblob(io)
|
|
58
|
+
when MysqlReplicator::Binlogs::FieldTypes::LONG_BLOB
|
|
59
|
+
parse_longblob(io)
|
|
60
|
+
when MysqlReplicator::Binlogs::FieldTypes::BINARY
|
|
61
|
+
parse_binary(io, column_def)
|
|
62
|
+
when MysqlReplicator::Binlogs::FieldTypes::VAR_BINARY
|
|
63
|
+
parse_varbinary(io, column_def)
|
|
64
|
+
when MysqlReplicator::Binlogs::FieldTypes::JSON
|
|
65
|
+
parse_json(io)
|
|
66
|
+
when MysqlReplicator::Binlogs::FieldTypes::ENUM
|
|
67
|
+
parse_enum(io, column_def)
|
|
68
|
+
else
|
|
69
|
+
raise MysqlReplicator::Error, "Unsupported type: #{type_code}"
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @rbs io: StringIO
|
|
74
|
+
# @rbs return: Integer
|
|
75
|
+
def self.parse_tinyint(io)
|
|
76
|
+
MysqlReplicator::StringIOUtil.read_int8(io)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @rbs io: StringIO
|
|
80
|
+
# @rbs return: Integer
|
|
81
|
+
def self.parse_smallint(io)
|
|
82
|
+
MysqlReplicator::StringIOUtil.read_int16(io)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# @rbs io: StringIO
|
|
86
|
+
# @rbs return: Integer
|
|
87
|
+
def self.parse_mediumint(io)
|
|
88
|
+
MysqlReplicator::StringIOUtil.read_int24(io)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# @rbs io: StringIO
|
|
92
|
+
# @rbs return: Integer
|
|
93
|
+
def self.parse_int(io)
|
|
94
|
+
MysqlReplicator::StringIOUtil.read_int32(io)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @rbs io: StringIO
|
|
98
|
+
# @rbs return: Integer
|
|
99
|
+
def self.parse_bigint(io)
|
|
100
|
+
MysqlReplicator::StringIOUtil.read_int64(io)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# @rbs io: StringIO
|
|
104
|
+
# @rbs return: Float
|
|
105
|
+
def self.parse_float(io)
|
|
106
|
+
MysqlReplicator::StringIOUtil.read_float32(io)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# @rbs io: StringIO
|
|
110
|
+
# @rbs return: Float
|
|
111
|
+
def self.parse_double(io)
|
|
112
|
+
MysqlReplicator::StringIOUtil.read_double64(io)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# This is sensitive...
|
|
116
|
+
#
|
|
117
|
+
# @rbs io: StringIO
|
|
118
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
119
|
+
# @rbs return: BigDecimal
|
|
120
|
+
def self.parse_decimal(io, column_def)
|
|
121
|
+
precision = column_def[:numeric_precision] || 10
|
|
122
|
+
scale = column_def[:numeric_scale] || 0
|
|
123
|
+
|
|
124
|
+
# Decimal format is integer and fractional parts
|
|
125
|
+
intg = precision - scale
|
|
126
|
+
intg_bytes = decimal_storage_bytes(intg)
|
|
127
|
+
frac_bytes = decimal_storage_bytes(scale)
|
|
128
|
+
total_bytes = intg_bytes + frac_bytes
|
|
129
|
+
|
|
130
|
+
data = MysqlReplicator::StringIOUtil.read_array_from_int8(io, total_bytes)
|
|
131
|
+
|
|
132
|
+
# top level bit is sign bit (1 = positive, 0 = negative)
|
|
133
|
+
negative = (data[0] & 0x80) == 0
|
|
134
|
+
# inversion of sign bit
|
|
135
|
+
data[0] ^= 0x80
|
|
136
|
+
data = data.map { |b| b ^ 0xFF } if negative
|
|
137
|
+
|
|
138
|
+
# parse integer part
|
|
139
|
+
intg_part = parse_decimal_digits(data[0, intg_bytes] || [], intg)
|
|
140
|
+
# parse fractional part
|
|
141
|
+
frac_part = parse_decimal_digits(data[intg_bytes, frac_bytes] || [], scale)
|
|
142
|
+
|
|
143
|
+
result = "#{intg_part}.#{frac_part.to_s.rjust(scale, '0')}"
|
|
144
|
+
result = "-#{result}" if negative
|
|
145
|
+
|
|
146
|
+
BigDecimal(result)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# @rbs io: StringIO
|
|
150
|
+
# @return: String
|
|
151
|
+
def self.parse_datetime(io)
|
|
152
|
+
# 5bytes if fractional seconds precision is 0
|
|
153
|
+
# format: 1bit sign + 17bits year*13month+month + 5bits day + 5bits hour + 6bits minute + 6bits second
|
|
154
|
+
data = MysqlReplicator::StringIOUtil.read_str(io, 5).unpack('C5').map(&:to_i)
|
|
155
|
+
value = (data[0] << 32) | (data[1] << 24) | (data[2] << 16) | (data[3] << 8) | data[4]
|
|
156
|
+
|
|
157
|
+
# top level bit is sign bit
|
|
158
|
+
value ^= 0x8000000000 # inversion of sign bit
|
|
159
|
+
|
|
160
|
+
second = value & 0x3F
|
|
161
|
+
value >>= 6
|
|
162
|
+
minute = value & 0x3F
|
|
163
|
+
value >>= 6
|
|
164
|
+
hour = value & 0x1F
|
|
165
|
+
value >>= 5
|
|
166
|
+
day = value & 0x1F
|
|
167
|
+
value >>= 5
|
|
168
|
+
year_month = value & 0x1FFFF
|
|
169
|
+
year = year_month / 13
|
|
170
|
+
month = year_month % 13
|
|
171
|
+
|
|
172
|
+
"#{year}-#{format('%02d', month)}-#{format('%02d', day)} " \
|
|
173
|
+
"#{format('%02d', hour)}:#{format('%02d', minute)}:#{format('%02d', second)}"
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# @rbs io: StringIO
|
|
177
|
+
# @return: String
|
|
178
|
+
def self.parse_date(io)
|
|
179
|
+
# 3bytes: YYYY*16*32 + MM*32 + DD
|
|
180
|
+
value = MysqlReplicator::StringIOUtil.read_uint24(io)
|
|
181
|
+
|
|
182
|
+
day = value & 0x1F
|
|
183
|
+
month = (value >> 5) & 0x0F
|
|
184
|
+
year = value >> 9
|
|
185
|
+
|
|
186
|
+
"#{year}-#{format('%02d', month)}-#{format('%02d', day)}"
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# @rbs io: StringIO
|
|
190
|
+
# @return: String
|
|
191
|
+
def self.parse_time(io)
|
|
192
|
+
# 3bytes if fractional seconds precision is 0
|
|
193
|
+
# format: 1bit sign + 1bit unused + 10bits hour + 6bits minute + 6bits second
|
|
194
|
+
data = MysqlReplicator::StringIOUtil.read_str(io, 3).unpack('C3').map(&:to_i)
|
|
195
|
+
value = (data[0] << 16) | (data[1] << 8) | data[2]
|
|
196
|
+
|
|
197
|
+
negative = (value & 0x800000) == 0
|
|
198
|
+
value &= 0x7FFFFF
|
|
199
|
+
|
|
200
|
+
hour = (value >> 12) & 0x3FF
|
|
201
|
+
minute = (value >> 6) & 0x3F
|
|
202
|
+
second = value & 0x3F
|
|
203
|
+
|
|
204
|
+
if negative
|
|
205
|
+
"-#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}"
|
|
206
|
+
else
|
|
207
|
+
"#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}"
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# @rbs io: StringIO
|
|
212
|
+
# @return: Integer
|
|
213
|
+
def self.parse_timestamp(io)
|
|
214
|
+
# 4bytes if fractional seconds precision is 0
|
|
215
|
+
# Unix Timestamp is Big-Engian
|
|
216
|
+
MysqlReplicator::StringIOUtil.read_uint32_big_endian(io)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# @rbs io: StringIO
|
|
220
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
221
|
+
# @rbs return: String
|
|
222
|
+
def self.parse_varchar(io, column_def)
|
|
223
|
+
max_length = column_def[:character_maximum_length] || 255
|
|
224
|
+
charset = column_def[:character_set_name]
|
|
225
|
+
|
|
226
|
+
# Determine length prefix size
|
|
227
|
+
bytes_per_char = case charset
|
|
228
|
+
when 'utf8mb4' then 4
|
|
229
|
+
when 'utf8', 'utf8mb3' then 3
|
|
230
|
+
else 1 # binary, latin1 and others
|
|
231
|
+
end
|
|
232
|
+
max_bytes = max_length * bytes_per_char
|
|
233
|
+
|
|
234
|
+
length = if max_bytes > 255
|
|
235
|
+
MysqlReplicator::StringIOUtil.read_uint16(io)
|
|
236
|
+
else
|
|
237
|
+
MysqlReplicator::StringIOUtil.read_uint8(io)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
value = MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
241
|
+
charset ? value.force_encoding('utf-8') : value
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
245
|
+
# @rbs return: String
|
|
246
|
+
def self.parse_char(io, column_def)
|
|
247
|
+
max_length = column_def[:character_maximum_length] || 10
|
|
248
|
+
charset = column_def[:character_set_name]
|
|
249
|
+
|
|
250
|
+
# Determine length prefix size
|
|
251
|
+
bytes_per_char = case charset
|
|
252
|
+
when 'utf8mb4' then 4
|
|
253
|
+
when 'utf8', 'utf8mb3' then 3
|
|
254
|
+
else 1 # binary, latin1 and others
|
|
255
|
+
end
|
|
256
|
+
max_bytes = max_length * bytes_per_char
|
|
257
|
+
|
|
258
|
+
length = if max_bytes > 255
|
|
259
|
+
MysqlReplicator::StringIOUtil.read_uint16(io)
|
|
260
|
+
else
|
|
261
|
+
MysqlReplicator::StringIOUtil.read_uint8(io)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
value = MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
265
|
+
charset ? value.force_encoding('utf-8') : value
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# @rbs io: StringIO
|
|
269
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
270
|
+
# @rbs return: String
|
|
271
|
+
def self.parse_tinytext(io, column_def)
|
|
272
|
+
charset = column_def[:character_set_name]
|
|
273
|
+
length = MysqlReplicator::StringIOUtil.read_uint8(io)
|
|
274
|
+
|
|
275
|
+
value = MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
276
|
+
charset ? value.force_encoding('utf-8') : value
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# @rbs io: StringIO
|
|
280
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
281
|
+
# @rbs return: String
|
|
282
|
+
def self.parse_text(io, column_def)
|
|
283
|
+
charset = column_def[:character_set_name]
|
|
284
|
+
length = MysqlReplicator::StringIOUtil.read_uint16(io)
|
|
285
|
+
|
|
286
|
+
value = MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
287
|
+
charset ? value.force_encoding('utf-8') : value
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# @rbs io: StringIO
|
|
291
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
292
|
+
# @rbs return: String
|
|
293
|
+
def self.parse_mediumtext(io, column_def)
|
|
294
|
+
charset = column_def[:character_set_name]
|
|
295
|
+
length = MysqlReplicator::StringIOUtil.read_uint24(io)
|
|
296
|
+
|
|
297
|
+
value = MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
298
|
+
charset ? value.force_encoding('utf-8') : value
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# @rbs io: StringIO
|
|
302
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
303
|
+
# @rbs return: String
|
|
304
|
+
def self.parse_longtext(io, column_def)
|
|
305
|
+
charset = column_def[:character_set_name]
|
|
306
|
+
length = MysqlReplicator::StringIOUtil.read_uint32(io)
|
|
307
|
+
|
|
308
|
+
value = MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
309
|
+
charset ? value.force_encoding('utf-8') : value
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# @rbs io: StringIO
|
|
313
|
+
# @rbs return: String
|
|
314
|
+
def self.parse_tinyblob(io)
|
|
315
|
+
length = MysqlReplicator::StringIOUtil.read_uint8(io)
|
|
316
|
+
MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# @rbs io: StringIO
|
|
320
|
+
# @rbs return: String
|
|
321
|
+
def self.parse_blob(io)
|
|
322
|
+
length = MysqlReplicator::StringIOUtil.read_uint16(io)
|
|
323
|
+
MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# @rbs io: StringIO
|
|
327
|
+
# @rbs return: String
|
|
328
|
+
def self.parse_mediumblob(io)
|
|
329
|
+
length = MysqlReplicator::StringIOUtil.read_uint24(io)
|
|
330
|
+
MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# @rbs io: StringIO
|
|
334
|
+
# @rbs return: String
|
|
335
|
+
def self.parse_longblob(io)
|
|
336
|
+
length = MysqlReplicator::StringIOUtil.read_uint32(io)
|
|
337
|
+
MysqlReplicator::StringIOUtil.read_str(io, length)
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# @rbs io: StringIO
|
|
341
|
+
# @rbs return: String
|
|
342
|
+
def self.parse_varbinary(io, column_def)
|
|
343
|
+
parse_varchar(io, column_def)
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# @rbs io: StringIO
|
|
347
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
348
|
+
# @rbs return: String
|
|
349
|
+
def self.parse_binary(io, column_def)
|
|
350
|
+
parse_char(io, column_def)
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# @rbs io: StringIO
|
|
354
|
+
# @return Hash[Symbol, String | Integer | bool | nil]
|
|
355
|
+
def self.parse_json(io)
|
|
356
|
+
length = MysqlReplicator::StringIOUtil.read_uint32(io)
|
|
357
|
+
data = io.read(length)
|
|
358
|
+
MysqlReplicator::Binlogs::JsonParser.parse(data)
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
# @rbs io: StringIO
|
|
362
|
+
# @rbs column_def: MysqlReplicator::Binlogs::TableMapEventParser::columnData
|
|
363
|
+
# @rbs return: String | Integer
|
|
364
|
+
def self.parse_enum(io, column_def)
|
|
365
|
+
enum_values = column_def[:enum_values] || []
|
|
366
|
+
index = if enum_values && enum_values.length > 255
|
|
367
|
+
MysqlReplicator::StringIOUtil.read_uint16(io)
|
|
368
|
+
else
|
|
369
|
+
MysqlReplicator::StringIOUtil.read_uint8(io)
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Enum index starts from 1
|
|
373
|
+
if index > 0 && enum_values
|
|
374
|
+
enum_values[index - 1]
|
|
375
|
+
else
|
|
376
|
+
index
|
|
377
|
+
end
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# @rbs digits: Integer
|
|
381
|
+
# @rbs return: Integer
|
|
382
|
+
def self.decimal_storage_bytes(digits)
|
|
383
|
+
# Each group of 9 digits takes 4 bytes
|
|
384
|
+
full_groups = digits / 9
|
|
385
|
+
leftover_digits = digits % 9
|
|
386
|
+
(full_groups * 4) + [0, 1, 1, 2, 2, 3, 3, 4, 4][leftover_digits]
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
# @rbs bytes: Array[Integer]
|
|
390
|
+
# @rbs digits: Integer
|
|
391
|
+
# @rbs return: Integer
|
|
392
|
+
def self.parse_decimal_digits(bytes, digits)
|
|
393
|
+
return 0 if digits == 0 || bytes.nil? || bytes.empty?
|
|
394
|
+
|
|
395
|
+
leftover_digits = digits % 9
|
|
396
|
+
leftover_bytes = [0, 1, 1, 2, 2, 3, 3, 4, 4][leftover_digits]
|
|
397
|
+
|
|
398
|
+
result = 0
|
|
399
|
+
offset = 0
|
|
400
|
+
|
|
401
|
+
# fraction part
|
|
402
|
+
if leftover_digits > 0
|
|
403
|
+
val = 0
|
|
404
|
+
leftover_bytes.times do |i|
|
|
405
|
+
val = (val << 8) | bytes[offset + i]
|
|
406
|
+
end
|
|
407
|
+
result = val
|
|
408
|
+
offset += leftover_bytes
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
# 9-digit groups (4 bytes each)
|
|
412
|
+
while offset < bytes.length
|
|
413
|
+
val = (bytes[offset] << 24) |
|
|
414
|
+
(bytes[offset + 1] << 16) |
|
|
415
|
+
(bytes[offset + 2] << 8) |
|
|
416
|
+
bytes[offset + 3]
|
|
417
|
+
result = (result * 1_000_000_000) + val.to_i
|
|
418
|
+
offset += 4
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
result
|
|
422
|
+
end
|
|
423
|
+
end
|
|
424
|
+
end
|
|
425
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module MysqlReplicator
|
|
5
|
+
module Binlogs
|
|
6
|
+
# MySQL Field Types contants
|
|
7
|
+
module FieldTypes
|
|
8
|
+
TINY_INT = 'tinyint' #: String
|
|
9
|
+
SMALL_INT = 'smallint' #: String
|
|
10
|
+
MEDIUM_INT = 'mediumint' #: String
|
|
11
|
+
INT = 'int' #: String
|
|
12
|
+
BIG_INT = 'bigint' #: String
|
|
13
|
+
FLOAT = 'float' #: String
|
|
14
|
+
DOUBLE = 'double' #: String
|
|
15
|
+
DECIMAL = 'decimal' #: String
|
|
16
|
+
DATETIME = 'datetime' #: String
|
|
17
|
+
DATE = 'date' #: String
|
|
18
|
+
TIME = 'time' #: String
|
|
19
|
+
TIMESTAMP = 'timestamp' #: String
|
|
20
|
+
CHAR = 'char' #: String
|
|
21
|
+
VARCHAR = 'varchar' #: String
|
|
22
|
+
TINY_TEXT = 'tinytext' #: String
|
|
23
|
+
MEDIUM_TEXT = 'mediumtext' #: String
|
|
24
|
+
TEXT = 'text' #: String
|
|
25
|
+
LONG_TEXT = 'longtext' #: String
|
|
26
|
+
TINY_BLOB = 'tinyblob' #: String
|
|
27
|
+
MEDIUM_BLOB = 'mediumblob' #: String
|
|
28
|
+
BLOB = 'blob' #: String
|
|
29
|
+
LONG_BLOB = 'longblob' #: String
|
|
30
|
+
BINARY = 'binary' #: String
|
|
31
|
+
VAR_BINARY = 'varbinary' #: String
|
|
32
|
+
JSON = 'json' #: String
|
|
33
|
+
ENUM = 'enum' #: String
|
|
34
|
+
UNKNOWN = 'unknown' #: String
|
|
35
|
+
|
|
36
|
+
# Mapping from INFORMATION_SCHEMA DATA_TYPE to MySQL Field Type
|
|
37
|
+
DATA_TYPE_MAP = {
|
|
38
|
+
'tinyint' => TINY_INT,
|
|
39
|
+
'smallint' => SMALL_INT,
|
|
40
|
+
'mediumint' => MEDIUM_INT,
|
|
41
|
+
'int' => INT,
|
|
42
|
+
'bigint' => BIG_INT,
|
|
43
|
+
'float' => FLOAT,
|
|
44
|
+
'double' => DOUBLE,
|
|
45
|
+
'decimal' => DECIMAL,
|
|
46
|
+
'datetime' => DATETIME,
|
|
47
|
+
'date' => DATE,
|
|
48
|
+
'time' => TIME,
|
|
49
|
+
'timestamp' => TIMESTAMP,
|
|
50
|
+
'char' => CHAR,
|
|
51
|
+
'varchar' => VARCHAR,
|
|
52
|
+
'tinytext' => TINY_TEXT,
|
|
53
|
+
'mediumtext' => MEDIUM_TEXT,
|
|
54
|
+
'text' => TEXT,
|
|
55
|
+
'longtext' => LONG_TEXT,
|
|
56
|
+
'tinyblob' => TINY_BLOB,
|
|
57
|
+
'mediumblob' => MEDIUM_BLOB,
|
|
58
|
+
'blob' => BLOB,
|
|
59
|
+
'longblob' => LONG_BLOB,
|
|
60
|
+
'binary' => BINARY,
|
|
61
|
+
'varbinary' => VAR_BINARY,
|
|
62
|
+
'enum' => ENUM,
|
|
63
|
+
'json' => JSON
|
|
64
|
+
}.freeze #: Hash[String, String]
|
|
65
|
+
|
|
66
|
+
# @rbs data_type: String
|
|
67
|
+
# @rbs return: String
|
|
68
|
+
def self.code_for(data_type)
|
|
69
|
+
base_data_type = data_type.downcase
|
|
70
|
+
DATA_TYPE_MAP[base_data_type] || UNKNOWN
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module MysqlReplicator
|
|
5
|
+
module Binlogs
|
|
6
|
+
class EventParser
|
|
7
|
+
# @rbs!
|
|
8
|
+
# type execution =
|
|
9
|
+
# MysqlReplicator::Binlogs::FormatDescriptionEventParser::execution |
|
|
10
|
+
# MysqlReplicator::Binlogs::QueryEventParser::execution |
|
|
11
|
+
# MysqlReplicator::Binlogs::RotateEventParser::execution |
|
|
12
|
+
# MysqlReplicator::Binlogs::RowsEventParser::execution |
|
|
13
|
+
# MysqlReplicator::Binlogs::TableMapEventParser::execution |
|
|
14
|
+
# MysqlReplicator::Binlogs::XidEventParser::execution |
|
|
15
|
+
# Hash[untyped, untyped]
|
|
16
|
+
|
|
17
|
+
# @rbs!
|
|
18
|
+
# type binlogEvent = {
|
|
19
|
+
# timestamp: Time,
|
|
20
|
+
# event_type: Symbol,
|
|
21
|
+
# server_id: Integer,
|
|
22
|
+
# length: Integer,
|
|
23
|
+
# next_position: Integer,
|
|
24
|
+
# flags: Integer,
|
|
25
|
+
# execution: execution
|
|
26
|
+
# }
|
|
27
|
+
|
|
28
|
+
# @rbs @stored_table_map: Hash[Integer, MysqlReplicator::Binlogs::TableMapEventParser::execution]
|
|
29
|
+
|
|
30
|
+
# @rbs return: void
|
|
31
|
+
def initialize
|
|
32
|
+
@stored_table_map = {}
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @rbs payload: String
|
|
36
|
+
# @rbs connection: MysqlReplicator::Connection
|
|
37
|
+
# @rbs checksum_enabled: bool
|
|
38
|
+
# @rbs return: binlogEvent
|
|
39
|
+
def execute(payload, connection, checksum_enabled)
|
|
40
|
+
offset = 0
|
|
41
|
+
|
|
42
|
+
timestamp = Time.at(MysqlReplicator::StringUtil.read_uint32(payload[offset, 4]))
|
|
43
|
+
offset += 4
|
|
44
|
+
event_type = readable_event_type(MysqlReplicator::StringUtil.read_uint8(payload[offset]))
|
|
45
|
+
offset += 1
|
|
46
|
+
server_id = MysqlReplicator::StringUtil.read_uint32(payload[offset, 4])
|
|
47
|
+
offset += 4
|
|
48
|
+
event_length = MysqlReplicator::StringUtil.read_uint32(payload[offset, 4])
|
|
49
|
+
offset += 4
|
|
50
|
+
next_position = MysqlReplicator::StringUtil.read_uint32(payload[offset, 4])
|
|
51
|
+
offset += 4
|
|
52
|
+
flags = MysqlReplicator::StringUtil.read_uint16(payload[offset, 2])
|
|
53
|
+
offset += 2
|
|
54
|
+
|
|
55
|
+
payload_length = event_length - offset
|
|
56
|
+
|
|
57
|
+
execution = parse_execution_data(
|
|
58
|
+
event_type,
|
|
59
|
+
MysqlReplicator::StringUtil.read_str(payload[offset, payload_length]),
|
|
60
|
+
connection,
|
|
61
|
+
checksum_enabled
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
{
|
|
65
|
+
timestamp: timestamp,
|
|
66
|
+
event_type: event_type,
|
|
67
|
+
server_id: server_id,
|
|
68
|
+
length: event_length,
|
|
69
|
+
next_position: next_position,
|
|
70
|
+
flags: flags,
|
|
71
|
+
execution: execution
|
|
72
|
+
}
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Basic event type identification
|
|
76
|
+
#
|
|
77
|
+
# @rbs event_type: Integer
|
|
78
|
+
# @rbs return: Symbol
|
|
79
|
+
def readable_event_type(event_type)
|
|
80
|
+
case event_type
|
|
81
|
+
when 2
|
|
82
|
+
:QUERY
|
|
83
|
+
when 4
|
|
84
|
+
:ROTATE
|
|
85
|
+
when 15
|
|
86
|
+
:FORMAT_DESCRIPTION
|
|
87
|
+
when 16
|
|
88
|
+
:XID
|
|
89
|
+
when 19
|
|
90
|
+
:TABLE_MAP
|
|
91
|
+
when 30
|
|
92
|
+
:WRITE_ROWS
|
|
93
|
+
when 31
|
|
94
|
+
:UPDATE_ROWS
|
|
95
|
+
when 32
|
|
96
|
+
:DELETE_ROWS
|
|
97
|
+
else
|
|
98
|
+
:UNKNOWN
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# @rbs event_type: Symbol
|
|
103
|
+
# @rbs payload: String
|
|
104
|
+
# @rbs connection: MysqlReplicator::Connection
|
|
105
|
+
# @rbs checksum_enabled: bool
|
|
106
|
+
# @rbs return: execution
|
|
107
|
+
def parse_execution_data(event_type, payload, connection, checksum_enabled)
|
|
108
|
+
case event_type
|
|
109
|
+
when :QUERY
|
|
110
|
+
MysqlReplicator::Binlogs::QueryEventParser.parse(payload, checksum_enabled)
|
|
111
|
+
when :ROTATE
|
|
112
|
+
MysqlReplicator::Binlogs::RotateEventParser.parse(payload, checksum_enabled)
|
|
113
|
+
when :FORMAT_DESCRIPTION
|
|
114
|
+
MysqlReplicator::Binlogs::FormatDescriptionEventParser.parse(payload)
|
|
115
|
+
when :TABLE_MAP
|
|
116
|
+
result = MysqlReplicator::Binlogs::TableMapEventParser.parse(payload, connection)
|
|
117
|
+
# Store in table map for future row events
|
|
118
|
+
@stored_table_map[result[:table_id]] = result
|
|
119
|
+
result
|
|
120
|
+
when :WRITE_ROWS
|
|
121
|
+
MysqlReplicator::Binlogs::RowsEventParser.parse(:WRITE_ROWS, payload, checksum_enabled, @stored_table_map)
|
|
122
|
+
when :UPDATE_ROWS
|
|
123
|
+
MysqlReplicator::Binlogs::RowsEventParser.parse(:UPDATE_ROWS, payload, checksum_enabled, @stored_table_map)
|
|
124
|
+
when :DELETE_ROWS
|
|
125
|
+
MysqlReplicator::Binlogs::RowsEventParser.parse(:DELETE_ROWS, payload, checksum_enabled, @stored_table_map)
|
|
126
|
+
when :XID
|
|
127
|
+
MysqlReplicator::Binlogs::XidEventParser.parse(payload)
|
|
128
|
+
else
|
|
129
|
+
{}
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module MysqlReplicator
|
|
5
|
+
module Binlogs
|
|
6
|
+
class FormatDescriptionEventParser
|
|
7
|
+
# @rbs!
|
|
8
|
+
# type execution = { binlog_version: Integer, server_version: String }
|
|
9
|
+
|
|
10
|
+
# @rbs payload: String
|
|
11
|
+
# @rbs return: execution
|
|
12
|
+
def self.parse(payload)
|
|
13
|
+
binlog_version = MysqlReplicator::StringUtil.read_uint16(payload[0, 2])
|
|
14
|
+
server_version = if payload.length >= 52
|
|
15
|
+
MysqlReplicator::StringUtil.read_str(payload[2, 50]).strip.gsub("\x00", '')
|
|
16
|
+
else
|
|
17
|
+
''
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
{ binlog_version: binlog_version, server_version: server_version }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|