mysql_binlog 0.3.2 → 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/mysql_binlog_dump +28 -29
- data/lib/mysql_binlog/binlog.rb +34 -1
- data/lib/mysql_binlog/binlog_event_parser.rb +171 -38
- data/lib/mysql_binlog/binlog_field_parser.rb +72 -7
- data/lib/mysql_binlog/reader/binlog_file_reader.rb +2 -2
- data/lib/mysql_binlog/version.rb +1 -1
- metadata +20 -42
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d4cb94c1699fea0fcfb90ef92d1d09d15dadb5a5
|
4
|
+
data.tar.gz: d6dcf06a12be0f42347344d250156f34a2ecf962
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7920efabe0e87c67d041d6712afeb2d27cd72978504e314ab4b9ee78fdeeaff2895e363c896a45972d3c0eeabd6ad84495c8f591013e137ed65893762111291b
|
7
|
+
data.tar.gz: c82e4f6b2fbf47de40505c70a310eca59e96f03c2cd85da4435cb69d582d80a3a597ea865ef43257547a927df83b03caa42b29bcc29f36b374b0e7a0f649b12c
|
data/bin/mysql_binlog_dump
CHANGED
@@ -15,13 +15,16 @@ def usage(exit_code, message = nil)
|
|
15
15
|
|
16
16
|
Usage:
|
17
17
|
To read from a binary log file on disk:
|
18
|
-
mysql_binlog_dump [options]
|
18
|
+
mysql_binlog_dump [options] <filename(s)>
|
19
19
|
|
20
20
|
--help, -?
|
21
21
|
Show this help.
|
22
22
|
|
23
23
|
--file, -f <filename>
|
24
|
-
Read from a binary log file on disk.
|
24
|
+
Read from a binary log file on disk (deprecated).
|
25
|
+
|
26
|
+
--checksum, -c
|
27
|
+
Enable CRC32 checksums.
|
25
28
|
|
26
29
|
--debug, -d
|
27
30
|
Debug reading from the binary log, showing calls into the reader and the
|
@@ -45,13 +48,16 @@ end
|
|
45
48
|
|
46
49
|
@options = OpenStruct.new
|
47
50
|
@options.file = nil
|
51
|
+
@options.checksum = nil
|
48
52
|
@options.debug = false
|
49
53
|
@options.tail = false
|
50
54
|
@options.rotate = false
|
55
|
+
@options.filenames = []
|
51
56
|
|
52
57
|
getopt_options = [
|
53
58
|
[ "--help", "-?", GetoptLong::NO_ARGUMENT ],
|
54
59
|
[ "--file", "-f", GetoptLong::REQUIRED_ARGUMENT ],
|
60
|
+
[ "--checksum", "-c", GetoptLong::NO_ARGUMENT ],
|
55
61
|
[ "--debug", "-d", GetoptLong::NO_ARGUMENT ],
|
56
62
|
[ "--tail", "-t", GetoptLong::NO_ARGUMENT ],
|
57
63
|
[ "--rotate", "-r", GetoptLong::NO_ARGUMENT ],
|
@@ -64,7 +70,9 @@ getopt.each do |opt, arg|
|
|
64
70
|
when "--help"
|
65
71
|
usage 0
|
66
72
|
when "--file"
|
67
|
-
@options.
|
73
|
+
@options.filenames << arg
|
74
|
+
when "--checksum"
|
75
|
+
@options.checksum = :crc32
|
68
76
|
when "--debug"
|
69
77
|
@options.debug = true
|
70
78
|
when "--tail"
|
@@ -74,34 +82,25 @@ getopt.each do |opt, arg|
|
|
74
82
|
end
|
75
83
|
end
|
76
84
|
|
77
|
-
|
78
|
-
usage 1, "A file must be provided with --file/-f"
|
79
|
-
end
|
85
|
+
@options.filenames.concat(ARGV)
|
80
86
|
|
81
|
-
|
82
|
-
|
83
|
-
reader = DebuggingReader.new(reader, :data => true, :calls => true)
|
87
|
+
if @options.filenames.empty?
|
88
|
+
usage 1, "One or more filenames must be provided"
|
84
89
|
end
|
85
|
-
binlog = Binlog.new(reader)
|
86
90
|
|
87
|
-
|
88
|
-
reader
|
89
|
-
|
90
|
-
|
91
|
-
end
|
91
|
+
@options.filenames.each do |filename|
|
92
|
+
reader = BinlogFileReader.new(filename)
|
93
|
+
if @options.debug
|
94
|
+
reader = DebuggingReader.new(reader, :data => true, :calls => true)
|
95
|
+
end
|
96
|
+
binlog = Binlog.new(reader)
|
97
|
+
binlog.checksum = @options.checksum
|
92
98
|
|
93
|
-
|
94
|
-
binlog.ignore_rotate =
|
95
|
-
else
|
96
|
-
binlog.ignore_rotate = true
|
97
|
-
end
|
99
|
+
reader.tail = @options.tail
|
100
|
+
binlog.ignore_rotate = !@options.rotate
|
98
101
|
|
99
|
-
binlog.each_event do |event|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
]
|
105
|
-
#pp event
|
106
|
-
#puts
|
107
|
-
end
|
102
|
+
binlog.each_event do |event|
|
103
|
+
pp event
|
104
|
+
puts
|
105
|
+
end
|
106
|
+
end
|
data/lib/mysql_binlog/binlog.rb
CHANGED
@@ -51,6 +51,7 @@ module MysqlBinlog
|
|
51
51
|
attr_accessor :filter_flags
|
52
52
|
attr_accessor :ignore_rotate
|
53
53
|
attr_accessor :max_query_length
|
54
|
+
attr_accessor :checksum
|
54
55
|
|
55
56
|
def initialize(reader)
|
56
57
|
@reader = reader
|
@@ -61,6 +62,7 @@ module MysqlBinlog
|
|
61
62
|
@filter_flags = nil
|
62
63
|
@ignore_rotate = false
|
63
64
|
@max_query_length = 1048576
|
65
|
+
@checksum = :nil
|
64
66
|
end
|
65
67
|
|
66
68
|
# Rewind to the beginning of the log, if supported by the reader. The
|
@@ -81,10 +83,16 @@ module MysqlBinlog
|
|
81
83
|
def read_event_fields(header)
|
82
84
|
# Delegate the parsing of the event content to a method of the same name
|
83
85
|
# in BinlogEventParser.
|
84
|
-
if event_parser.methods.include? header[:event_type]
|
86
|
+
if event_parser.methods.map(&:to_sym).include? header[:event_type]
|
85
87
|
fields = event_parser.send(header[:event_type], header)
|
86
88
|
end
|
87
89
|
|
90
|
+
unless fields
|
91
|
+
fields = {
|
92
|
+
payload: reader.read(header[:payload_length]),
|
93
|
+
}
|
94
|
+
end
|
95
|
+
|
88
96
|
# Check if we've read past the end of the event. This is normally because
|
89
97
|
# of an unsupported substructure in the event causing field misalignment
|
90
98
|
# or a bug in the event reader method in BinlogEventParser. This may also
|
@@ -103,6 +111,19 @@ module MysqlBinlog
|
|
103
111
|
end
|
104
112
|
private :read_event_fields
|
105
113
|
|
114
|
+
def checksum_length
|
115
|
+
case @checksum
|
116
|
+
when :crc32
|
117
|
+
4
|
118
|
+
else
|
119
|
+
0
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def payload_length(header)
|
124
|
+
@fde ? (header[:event_length] - @fde[:header_length] - checksum_length) : 0
|
125
|
+
end
|
126
|
+
|
106
127
|
# Scan events until finding one that isn't rejected by the filter rules.
|
107
128
|
# If there are no filter rules, this will return the next event provided
|
108
129
|
# by the reader.
|
@@ -119,6 +140,18 @@ module MysqlBinlog
|
|
119
140
|
return nil
|
120
141
|
end
|
121
142
|
|
143
|
+
# Skip the remaining part of the header which might not have been
|
144
|
+
# parsed.
|
145
|
+
if @fde
|
146
|
+
reader.seek(position + @fde[:header_length])
|
147
|
+
header[:payload_length] = payload_length(header)
|
148
|
+
header[:payload_end] = position + @fde[:header_length] + payload_length(header)
|
149
|
+
else
|
150
|
+
header[:payload_length] = 0
|
151
|
+
header[:payload_end] = header[:next_position]
|
152
|
+
end
|
153
|
+
|
154
|
+
|
122
155
|
if @filter_event_types
|
123
156
|
unless @filter_event_types.include? header[:event_type]
|
124
157
|
skip_this_event = true
|
@@ -26,11 +26,23 @@ module MysqlBinlog
|
|
26
26
|
:pre_ga_write_rows_event => 20, # (deprecated)
|
27
27
|
:pre_ga_update_rows_event => 21, # (deprecated)
|
28
28
|
:pre_ga_delete_rows_event => 22, # (deprecated)
|
29
|
-
:
|
30
|
-
:
|
31
|
-
:
|
29
|
+
:write_rows_event_v1 => 23, #
|
30
|
+
:update_rows_event_v1 => 24, #
|
31
|
+
:delete_rows_event_v1 => 25, #
|
32
32
|
:incident_event => 26, #
|
33
33
|
:heartbeat_log_event => 27, #
|
34
|
+
:ignorable_log_event => 28,
|
35
|
+
:rows_query_log_event => 29,
|
36
|
+
:write_rows_event_v2 => 30,
|
37
|
+
:update_rows_event_v2 => 31,
|
38
|
+
:delete_rows_event_v2 => 32,
|
39
|
+
:gtid_log_event => 33,
|
40
|
+
:anonymous_gtid_log_event => 34,
|
41
|
+
:previous_gtids_log_event => 35,
|
42
|
+
:transaction_context_event => 36,
|
43
|
+
:view_change_event => 37,
|
44
|
+
:xa_prepare_log_event => 38,
|
45
|
+
|
34
46
|
:table_metadata_event => 50, # Only in Twitter MySQL
|
35
47
|
}
|
36
48
|
|
@@ -44,9 +56,9 @@ module MysqlBinlog
|
|
44
56
|
# have an identical structure, this list can be used by other programs to
|
45
57
|
# know which events can be treated as row events.
|
46
58
|
ROW_EVENT_TYPES = [
|
47
|
-
:
|
48
|
-
:
|
49
|
-
:
|
59
|
+
:write_rows_event_v1,
|
60
|
+
:update_rows_event_v1,
|
61
|
+
:delete_rows_event_v1,
|
50
62
|
]
|
51
63
|
|
52
64
|
# Values for the +flags+ field that may appear in binary logs. There are
|
@@ -55,11 +67,14 @@ module MysqlBinlog
|
|
55
67
|
#
|
56
68
|
# Defined in sql/log_event.h line ~448
|
57
69
|
EVENT_HEADER_FLAGS = {
|
58
|
-
:binlog_in_use
|
59
|
-
:thread_specific
|
60
|
-
:suppress_use
|
61
|
-
:artificial
|
62
|
-
:relay_log
|
70
|
+
:binlog_in_use => 0x0001, # LOG_EVENT_BINLOG_IN_USE_F
|
71
|
+
:thread_specific => 0x0004, # LOG_EVENT_THREAD_SPECIFIC_F
|
72
|
+
:suppress_use => 0x0008, # LOG_EVENT_SUPPRESS_USE_F
|
73
|
+
:artificial => 0x0020, # LOG_EVENT_ARTIFICIAL_F
|
74
|
+
:relay_log => 0x0040, # LOG_EVENT_RELAY_LOG_F
|
75
|
+
:ignorable => 0x0080, # LOG_EVENT_IGNORABLE_F
|
76
|
+
:no_filter => 0x0100, # LOG_EVENT_NO_FILTER_F
|
77
|
+
:mts_isolate => 0x0200, # LOG_EVENT_MTS_ISOLATE_F
|
63
78
|
}
|
64
79
|
|
65
80
|
# A mapping array for all values that may appear in the +status+ field of
|
@@ -79,8 +94,15 @@ module MysqlBinlog
|
|
79
94
|
:table_map_for_update, # 9 (Q_TABLE_MAP_FOR_UPDATE_CODE)
|
80
95
|
:master_data_written, # 10 (Q_MASTER_DATA_WRITTEN_CODE)
|
81
96
|
:invoker, # 11 (Q_INVOKER)
|
97
|
+
:updated_db_names, # 12 (Q_UPDATED_DB_NAMES)
|
98
|
+
:microseconds, # 13 (Q_MICROSECONDS)
|
99
|
+
:commit_ts, # 14 (Q_COMMIT_TS)
|
100
|
+
:commit_ts2, # 15
|
101
|
+
:explicit_defaults_for_timestamp, # 16
|
82
102
|
]
|
83
103
|
|
104
|
+
QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS = 254
|
105
|
+
|
84
106
|
# A mapping hash for all values that may appear in the +flags2+ field of
|
85
107
|
# a query_event.
|
86
108
|
#
|
@@ -181,11 +203,13 @@ module MysqlBinlog
|
|
181
203
|
def event_header
|
182
204
|
header = {}
|
183
205
|
header[:timestamp] = parser.read_uint32
|
184
|
-
|
206
|
+
event_type = parser.read_uint8
|
207
|
+
header[:event_type] = EVENT_TYPES[event_type] || "unknown_#{event_type}".to_sym
|
185
208
|
header[:server_id] = parser.read_uint32
|
186
209
|
header[:event_length] = parser.read_uint32
|
187
210
|
header[:next_position] = parser.read_uint32
|
188
211
|
header[:flags] = parser.read_uint_bitmap_by_size_and_name(2, EVENT_HEADER_FLAGS)
|
212
|
+
|
189
213
|
header
|
190
214
|
end
|
191
215
|
|
@@ -212,6 +236,25 @@ module MysqlBinlog
|
|
212
236
|
fields
|
213
237
|
end
|
214
238
|
|
239
|
+
def _query_event_status_updated_db_names
|
240
|
+
db_count = parser.read_uint8
|
241
|
+
return nil if db_count == QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS
|
242
|
+
|
243
|
+
db_names = []
|
244
|
+
db_count.times do |n|
|
245
|
+
db_name = ""
|
246
|
+
loop do
|
247
|
+
c = reader.read(1)
|
248
|
+
break if c == "\0"
|
249
|
+
db_name << c
|
250
|
+
end
|
251
|
+
db_names << db_name
|
252
|
+
end
|
253
|
+
|
254
|
+
db_names
|
255
|
+
end
|
256
|
+
private :_query_event_status_updated_db_names
|
257
|
+
|
215
258
|
# Parse a dynamic +status+ structure within a query_event, which consists
|
216
259
|
# of a status_length (uint16) followed by a number of status variables
|
217
260
|
# (determined by the +status_length+) each of which consist of:
|
@@ -223,7 +266,8 @@ module MysqlBinlog
|
|
223
266
|
status_length = parser.read_uint16
|
224
267
|
end_position = reader.position + status_length
|
225
268
|
while reader.position < end_position
|
226
|
-
|
269
|
+
status_type_id = parser.read_uint8
|
270
|
+
status_type = QUERY_EVENT_STATUS_TYPES[status_type_id]
|
227
271
|
status[status_type] = case status_type
|
228
272
|
when :flags2
|
229
273
|
parser.read_uint_bitmap_by_size_and_name(4, QUERY_EVENT_FLAGS2)
|
@@ -252,6 +296,12 @@ module MysqlBinlog
|
|
252
296
|
parser.read_uint16
|
253
297
|
when :table_map_for_update
|
254
298
|
parser.read_uint64
|
299
|
+
when :updated_db_names
|
300
|
+
_query_event_status_updated_db_names
|
301
|
+
when :commit_ts
|
302
|
+
parser.read_uint64
|
303
|
+
else
|
304
|
+
raise "Unknown status type #{status_type_id}"
|
255
305
|
end
|
256
306
|
end
|
257
307
|
|
@@ -334,7 +384,7 @@ module MysqlBinlog
|
|
334
384
|
:precision => parser.read_uint8,
|
335
385
|
:decimals => parser.read_uint8,
|
336
386
|
}
|
337
|
-
when :blob, :geometry
|
387
|
+
when :blob, :geometry, :json
|
338
388
|
{ :length_size => parser.read_uint8 }
|
339
389
|
when :string, :var_string
|
340
390
|
# The :string type sets a :real_type field to indicate the actual type
|
@@ -350,6 +400,10 @@ module MysqlBinlog
|
|
350
400
|
else
|
351
401
|
{ :max_length => (((metadata >> 4) & 0x300) ^ 0x300) + (metadata & 0x00ff) }
|
352
402
|
end
|
403
|
+
when :timestamp2, :datetime2, :time2
|
404
|
+
{
|
405
|
+
:decimals => parser.read_uint8,
|
406
|
+
}
|
353
407
|
end
|
354
408
|
end
|
355
409
|
private :_table_map_event_column_metadata_read
|
@@ -376,7 +430,7 @@ module MysqlBinlog
|
|
376
430
|
map_entry[:db] = parser.read_lpstringz
|
377
431
|
map_entry[:table] = parser.read_lpstringz
|
378
432
|
columns = parser.read_varint
|
379
|
-
columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] }
|
433
|
+
columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] || "unknown_#{c}".to_sym }
|
380
434
|
columns_metadata = _table_map_event_column_metadata(columns_type)
|
381
435
|
columns_nullable = parser.read_bit_array(columns)
|
382
436
|
|
@@ -415,8 +469,9 @@ module MysqlBinlog
|
|
415
469
|
fields[:flags] = parser.read_uint16
|
416
470
|
fields[:columns] = columns.times.map do |c|
|
417
471
|
descriptor_length = parser.read_uint32
|
472
|
+
column_type = parser.read_uint8
|
418
473
|
@table_map[table_id][:columns][c][:description] = {
|
419
|
-
:type => MYSQL_TYPES[
|
474
|
+
:type => MYSQL_TYPES[column_type] || "unknown_#{column_type}".to_sym,
|
420
475
|
:length => parser.read_uint32,
|
421
476
|
:scale => parser.read_uint8,
|
422
477
|
:character_set => COLLATION[parser.read_uint16],
|
@@ -433,41 +488,62 @@ module MysqlBinlog
|
|
433
488
|
# Parse a single row image, which is comprised of a series of columns. Not
|
434
489
|
# all columns are present in the row image, the columns_used array of true
|
435
490
|
# and false values identifies which columns are present.
|
436
|
-
def
|
491
|
+
def _generic_rows_event_row_image_v1(header, fields, columns_used)
|
437
492
|
row_image = []
|
493
|
+
start_position = reader.position
|
438
494
|
columns_null = parser.read_bit_array(fields[:table][:columns].size)
|
439
495
|
fields[:table][:columns].each_with_index do |column, column_index|
|
496
|
+
#puts "column #{column_index} #{column}: used=#{columns_used[column_index]}, null=#{columns_null[column_index]}"
|
440
497
|
if !columns_used[column_index]
|
441
498
|
row_image << nil
|
442
499
|
elsif columns_null[column_index]
|
443
500
|
row_image << { column_index => nil }
|
444
501
|
else
|
502
|
+
value = parser.read_mysql_type(column[:type], column[:metadata])
|
445
503
|
row_image << {
|
446
|
-
column_index =>
|
447
|
-
parser.read_mysql_type(column[:type], column[:metadata])
|
504
|
+
column_index => value,
|
448
505
|
}
|
449
506
|
end
|
450
507
|
end
|
451
|
-
|
508
|
+
end_position = reader.position
|
509
|
+
|
510
|
+
{
|
511
|
+
image: row_image,
|
512
|
+
size: end_position-start_position
|
513
|
+
}
|
514
|
+
end
|
515
|
+
private :_generic_rows_event_row_image_v1
|
516
|
+
|
517
|
+
def diff_row_images(before, after)
|
518
|
+
diff = {}
|
519
|
+
before.each_with_index do |before_column, index|
|
520
|
+
after_column = after[index]
|
521
|
+
before_value = before_column.first[1]
|
522
|
+
after_value = after_column.first[1]
|
523
|
+
if before_value != after_value
|
524
|
+
diff[index] = { before: before_value, after: after_value }
|
525
|
+
end
|
526
|
+
end
|
527
|
+
diff
|
452
528
|
end
|
453
|
-
private :_generic_rows_event_row_image
|
454
529
|
|
455
530
|
# Parse the row images present in a row-based replication row event. This
|
456
531
|
# is rather incomplete right now due missing support for many MySQL types,
|
457
532
|
# but can parse some basic events.
|
458
|
-
def
|
533
|
+
def _generic_rows_event_row_images_v1(header, fields, columns_used)
|
459
534
|
row_images = []
|
460
535
|
end_position = reader.position + reader.remaining(header)
|
461
536
|
while reader.position < end_position
|
462
537
|
row_image = {}
|
463
538
|
case header[:event_type]
|
464
|
-
when :
|
465
|
-
row_image[:after] =
|
466
|
-
when :
|
467
|
-
row_image[:before] =
|
468
|
-
when :
|
469
|
-
row_image[:before] =
|
470
|
-
row_image[:after] =
|
539
|
+
when :write_rows_event_v1
|
540
|
+
row_image[:after] = _generic_rows_event_row_image_v1(header, fields, columns_used[:after])
|
541
|
+
when :delete_rows_event_v1
|
542
|
+
row_image[:before] = _generic_rows_event_row_image_v1(header, fields, columns_used[:before])
|
543
|
+
when :update_rows_event_v1
|
544
|
+
row_image[:before] = _generic_rows_event_row_image_v1(header, fields, columns_used[:before])
|
545
|
+
row_image[:after] = _generic_rows_event_row_image_v1(header, fields, columns_used[:after])
|
546
|
+
row_image[:diff] = diff_row_images(row_image[:before][:image], row_image[:after][:image])
|
471
547
|
end
|
472
548
|
row_images << row_image
|
473
549
|
end
|
@@ -481,7 +557,7 @@ module MysqlBinlog
|
|
481
557
|
|
482
558
|
row_images
|
483
559
|
end
|
484
|
-
private :
|
560
|
+
private :_generic_rows_event_row_images_v1
|
485
561
|
|
486
562
|
# Parse fields for any of the row-based replication row events:
|
487
563
|
# * +Write_rows+ which is used for +INSERT+.
|
@@ -491,7 +567,7 @@ module MysqlBinlog
|
|
491
567
|
# Implemented in sql/log_event.cc line ~8039
|
492
568
|
# in Rows_log_event::write_data_header
|
493
569
|
# and Rows_log_event::write_data_body
|
494
|
-
def
|
570
|
+
def generic_rows_event_v1(header)
|
495
571
|
fields = {}
|
496
572
|
table_id = parser.read_uint48
|
497
573
|
fields[:table] = @table_map[table_id]
|
@@ -499,21 +575,78 @@ module MysqlBinlog
|
|
499
575
|
columns = parser.read_varint
|
500
576
|
columns_used = {}
|
501
577
|
case header[:event_type]
|
502
|
-
when :
|
578
|
+
when :write_rows_event_v1
|
503
579
|
columns_used[:after] = parser.read_bit_array(columns)
|
504
|
-
when :
|
580
|
+
when :delete_rows_event_v1
|
505
581
|
columns_used[:before] = parser.read_bit_array(columns)
|
506
|
-
when :
|
582
|
+
when :update_rows_event_v1
|
507
583
|
columns_used[:before] = parser.read_bit_array(columns)
|
508
584
|
columns_used[:after] = parser.read_bit_array(columns)
|
509
585
|
end
|
510
|
-
fields[:row_image] =
|
586
|
+
fields[:row_image] = _generic_rows_event_row_images_v1(header, fields, columns_used)
|
511
587
|
fields
|
512
588
|
end
|
513
589
|
|
514
|
-
alias :
|
515
|
-
alias :
|
516
|
-
alias :
|
590
|
+
alias :write_rows_event_v1 :generic_rows_event_v1
|
591
|
+
alias :update_rows_event_v1 :generic_rows_event_v1
|
592
|
+
alias :delete_rows_event_v1 :generic_rows_event_v1
|
593
|
+
|
594
|
+
def rows_query_log_event(header)
|
595
|
+
reader.read(1) # skip useless byte length which is unused
|
596
|
+
{ query: reader.read(header[:payload_length]-1) }
|
597
|
+
end
|
598
|
+
|
599
|
+
def in_hex(bytes)
|
600
|
+
bytes.each_byte.map { |c| "%02x" % c.ord }.join
|
601
|
+
end
|
602
|
+
|
603
|
+
def format_gtid_sid(sid)
|
604
|
+
[0..3, 4..5, 6..7, 8..9, 10..15].map { |r| in_hex(sid[r]) }.join("-")
|
605
|
+
end
|
517
606
|
|
607
|
+
# 6d9190a2-cca6-11e8-aa8c-42010aef0019:551845019
|
608
|
+
def format_gtid(sid, gno_or_ivs)
|
609
|
+
"#{format_gtid_sid(sid)}:#{gno_or_ivs}"
|
610
|
+
end
|
611
|
+
|
612
|
+
def previous_gtids_log_event(header)
|
613
|
+
n_sids = parser.read_uint64
|
614
|
+
|
615
|
+
gtids = []
|
616
|
+
n_sids.times do
|
617
|
+
sid = parser.read_nstring(16)
|
618
|
+
n_ivs = parser.read_uint64
|
619
|
+
ivs = []
|
620
|
+
n_ivs.times do
|
621
|
+
iv_start = parser.read_uint64
|
622
|
+
iv_end = parser.read_uint64
|
623
|
+
ivs << "#{iv_start}-#{iv_end}"
|
624
|
+
end
|
625
|
+
gtids << format_gtid(sid, ivs.join(":"))
|
626
|
+
end
|
627
|
+
|
628
|
+
{
|
629
|
+
previous_gtids: gtids
|
630
|
+
}
|
631
|
+
end
|
632
|
+
|
633
|
+
def gtid_log_event(header)
|
634
|
+
flags = parser.read_uint8
|
635
|
+
sid = parser.read_nstring(16)
|
636
|
+
gno = parser.read_uint64
|
637
|
+
lts_type = parser.read_uint8
|
638
|
+
lts_last_committed = parser.read_uint64
|
639
|
+
lts_sequence_number = parser.read_uint64
|
640
|
+
|
641
|
+
{
|
642
|
+
flags: flags,
|
643
|
+
gtid: format_gtid(sid, gno),
|
644
|
+
lts: {
|
645
|
+
type: lts_type,
|
646
|
+
last_committed: lts_last_committed,
|
647
|
+
sequence_number: lts_sequence_number,
|
648
|
+
},
|
649
|
+
}
|
650
|
+
end
|
518
651
|
end
|
519
652
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'bigdecimal'
|
2
|
+
|
1
3
|
module MysqlBinlog
|
2
4
|
# All MySQL types mapping to their integer values.
|
3
5
|
MYSQL_TYPES_HASH = {
|
@@ -18,6 +20,10 @@ module MysqlBinlog
|
|
18
20
|
:newdate => 14,
|
19
21
|
:varchar => 15,
|
20
22
|
:bit => 16,
|
23
|
+
:timestamp2 => 17,
|
24
|
+
:datetime2 => 18,
|
25
|
+
:time2 => 19,
|
26
|
+
:json => 245,
|
21
27
|
:newdecimal => 246,
|
22
28
|
:enum => 247,
|
23
29
|
:set => 248,
|
@@ -64,6 +70,17 @@ module MysqlBinlog
|
|
64
70
|
a + (b << 8) + (c << 16)
|
65
71
|
end
|
66
72
|
|
73
|
+
# Read an unsigned 24-bit (3-byte) big-endian integer.
|
74
|
+
def read_uint24_be
|
75
|
+
a, b = reader.read(3).unpack("nC")
|
76
|
+
(a << 8) + b
|
77
|
+
end
|
78
|
+
|
79
|
+
# Read an unsigned 32-bit (4-byte) integer.
|
80
|
+
def read_uint32_be
|
81
|
+
reader.read(4).unpack("N").first
|
82
|
+
end
|
83
|
+
|
67
84
|
# Read an unsigned 32-bit (4-byte) integer.
|
68
85
|
def read_uint32
|
69
86
|
reader.read(4).unpack("V").first
|
@@ -75,6 +92,12 @@ module MysqlBinlog
|
|
75
92
|
a + (b << 8)
|
76
93
|
end
|
77
94
|
|
95
|
+
# Read an unsigned 40-bit (5-byte) big-endian integer.
|
96
|
+
def read_uint40_be
|
97
|
+
a, b = reader.read(5).unpack("NC")
|
98
|
+
(a << 8) + b
|
99
|
+
end
|
100
|
+
|
78
101
|
# Read an unsigned 48-bit (6-byte) integer.
|
79
102
|
def read_uint48
|
80
103
|
a, b, c = reader.read(6).unpack("vvv")
|
@@ -89,7 +112,12 @@ module MysqlBinlog
|
|
89
112
|
|
90
113
|
# Read an unsigned 64-bit (8-byte) integer.
|
91
114
|
def read_uint64
|
92
|
-
reader.read(8).unpack("Q").first
|
115
|
+
reader.read(8).unpack("Q<").first
|
116
|
+
end
|
117
|
+
|
118
|
+
# Read an unsigned 64-bit (8-byte) integer.
|
119
|
+
def read_uint64_be
|
120
|
+
reader.read(8).unpack("Q>").first
|
93
121
|
end
|
94
122
|
|
95
123
|
# Read a signed 8-bit (1-byte) integer.
|
@@ -99,13 +127,13 @@ module MysqlBinlog
|
|
99
127
|
|
100
128
|
# Read a signed 16-bit (2-byte) big-endian integer.
|
101
129
|
def read_int16_be
|
102
|
-
reader.read(2).
|
130
|
+
reader.read(2).unpack('n').first
|
103
131
|
end
|
104
132
|
|
105
133
|
# Read a signed 24-bit (3-byte) big-endian integer.
|
106
134
|
def read_int24_be
|
107
135
|
a, b, c = reader.read(3).unpack('CCC')
|
108
|
-
if a & 128
|
136
|
+
if (a & 128) == 0
|
109
137
|
(a << 16) | (b << 8) | c
|
110
138
|
else
|
111
139
|
(-1 << 24) | (a << 16) | (b << 8) | c
|
@@ -114,9 +142,9 @@ module MysqlBinlog
|
|
114
142
|
|
115
143
|
# Read a signed 32-bit (4-byte) big-endian integer.
|
116
144
|
def read_int32_be
|
117
|
-
reader.read(4).
|
145
|
+
reader.read(4).unpack('N').first
|
118
146
|
end
|
119
|
-
|
147
|
+
|
120
148
|
def read_uint_by_size(size)
|
121
149
|
case size
|
122
150
|
when 1
|
@@ -272,7 +300,7 @@ module MysqlBinlog
|
|
272
300
|
str << value.to_s
|
273
301
|
end
|
274
302
|
|
275
|
-
BigDecimal
|
303
|
+
BigDecimal(str)
|
276
304
|
end
|
277
305
|
|
278
306
|
# Read an array of unsigned 8-bit (1-byte) integers.
|
@@ -359,6 +387,41 @@ module MysqlBinlog
|
|
359
387
|
]
|
360
388
|
end
|
361
389
|
|
390
|
+
def convert_mysql_type_datetimef(int_part, frac_part)
|
391
|
+
year_month = extract_bits(int_part, 17, 22)
|
392
|
+
year = year_month / 13
|
393
|
+
month = year_month % 13
|
394
|
+
day = extract_bits(int_part, 5, 17)
|
395
|
+
hour = extract_bits(int_part, 5, 12)
|
396
|
+
minute = extract_bits(int_part, 6, 6)
|
397
|
+
second = extract_bits(int_part, 6, 0)
|
398
|
+
|
399
|
+
"%04i-%02i-%02i %02i:%02i:%02i.%06i" % [
|
400
|
+
year,
|
401
|
+
month,
|
402
|
+
day,
|
403
|
+
hour,
|
404
|
+
minute,
|
405
|
+
second,
|
406
|
+
frac_part,
|
407
|
+
]
|
408
|
+
end
|
409
|
+
|
410
|
+
def read_datetimef(decimals)
|
411
|
+
int_part = read_uint40_be
|
412
|
+
frac_part = case decimals
|
413
|
+
when 0
|
414
|
+
0
|
415
|
+
when 1, 2
|
416
|
+
read_uint8 * 10000
|
417
|
+
when 3, 4
|
418
|
+
read_uint16_be * 100
|
419
|
+
when 5, 6
|
420
|
+
read_uint24_be
|
421
|
+
end
|
422
|
+
convert_mysql_type_datetimef(int_part, frac_part)
|
423
|
+
end
|
424
|
+
|
362
425
|
# Read a single field, provided the MySQL column type as a symbol. Not all
|
363
426
|
# types are currently supported.
|
364
427
|
def read_mysql_type(type, metadata=nil)
|
@@ -382,7 +445,7 @@ module MysqlBinlog
|
|
382
445
|
when :varchar, :string
|
383
446
|
prefix_size = (metadata[:max_length] > 255) ? 2 : 1
|
384
447
|
read_lpstring(prefix_size)
|
385
|
-
when :blob, :geometry
|
448
|
+
when :blob, :geometry, :json
|
386
449
|
read_lpstring(metadata[:length_size])
|
387
450
|
when :timestamp
|
388
451
|
read_uint32
|
@@ -394,6 +457,8 @@ module MysqlBinlog
|
|
394
457
|
convert_mysql_type_time(read_uint24)
|
395
458
|
when :datetime
|
396
459
|
convert_mysql_type_datetime(read_uint64)
|
460
|
+
when :datetime2
|
461
|
+
read_datetimef(metadata[:decimals])
|
397
462
|
when :enum, :set
|
398
463
|
read_uint_by_size(metadata[:size])
|
399
464
|
when :bit
|
@@ -20,7 +20,7 @@ module MysqlBinlog
|
|
20
20
|
def open_file(filename)
|
21
21
|
@dirname = File.dirname(filename)
|
22
22
|
@filename = File.basename(filename)
|
23
|
-
@binlog = File.open(filename,
|
23
|
+
@binlog = File.open(filename, "r:BINARY")
|
24
24
|
|
25
25
|
verify_magic
|
26
26
|
end
|
@@ -69,7 +69,7 @@ module MysqlBinlog
|
|
69
69
|
end
|
70
70
|
|
71
71
|
def remaining(header)
|
72
|
-
header[:
|
72
|
+
header[:payload_end] - @binlog.tell
|
73
73
|
end
|
74
74
|
|
75
75
|
def skip(header)
|
data/lib/mysql_binlog/version.rb
CHANGED
metadata
CHANGED
@@ -1,32 +1,23 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: mysql_binlog
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
prerelease:
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 3
|
9
|
-
- 2
|
10
|
-
version: 0.3.2
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.3
|
11
5
|
platform: ruby
|
12
|
-
authors:
|
6
|
+
authors:
|
13
7
|
- Jeremy Cole
|
14
8
|
autorequire:
|
15
9
|
bindir: bin
|
16
10
|
cert_chain: []
|
17
|
-
|
18
|
-
date: 2012-11-07 00:00:00 Z
|
11
|
+
date: 2019-07-13 00:00:00.000000000 Z
|
19
12
|
dependencies: []
|
20
|
-
|
21
13
|
description: Library for parsing MySQL binary logs in Ruby
|
22
14
|
email: jeremy@jcole.us
|
23
|
-
executables:
|
15
|
+
executables:
|
24
16
|
- mysql_binlog_dump
|
25
17
|
extensions: []
|
26
|
-
|
27
18
|
extra_rdoc_files: []
|
28
|
-
|
29
|
-
|
19
|
+
files:
|
20
|
+
- bin/mysql_binlog_dump
|
30
21
|
- lib/mysql_binlog.rb
|
31
22
|
- lib/mysql_binlog/binlog.rb
|
32
23
|
- lib/mysql_binlog/binlog_event_parser.rb
|
@@ -36,40 +27,27 @@ files:
|
|
36
27
|
- lib/mysql_binlog/reader/binlog_stream_reader.rb
|
37
28
|
- lib/mysql_binlog/reader/debugging_reader.rb
|
38
29
|
- lib/mysql_binlog/version.rb
|
39
|
-
- bin/mysql_binlog_dump
|
40
30
|
homepage: http://jcole.us/
|
41
31
|
licenses: []
|
42
|
-
|
32
|
+
metadata: {}
|
43
33
|
post_install_message:
|
44
34
|
rdoc_options: []
|
45
|
-
|
46
|
-
require_paths:
|
35
|
+
require_paths:
|
47
36
|
- lib
|
48
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
49
|
-
|
50
|
-
requirements:
|
37
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
51
39
|
- - ">="
|
52
|
-
- !ruby/object:Gem::Version
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
version: "0"
|
57
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
58
|
-
none: false
|
59
|
-
requirements:
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
60
44
|
- - ">="
|
61
|
-
- !ruby/object:Gem::Version
|
62
|
-
|
63
|
-
segments:
|
64
|
-
- 0
|
65
|
-
version: "0"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
66
47
|
requirements: []
|
67
|
-
|
68
48
|
rubyforge_project:
|
69
|
-
rubygems_version:
|
49
|
+
rubygems_version: 2.5.2.3
|
70
50
|
signing_key:
|
71
|
-
specification_version:
|
51
|
+
specification_version: 4
|
72
52
|
summary: MySQL Binary Log Parser
|
73
53
|
test_files: []
|
74
|
-
|
75
|
-
has_rdoc:
|