mysql_binlog 0.3.2 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 68751dcb93a315960a1ea2ec299a9f26eef45678d58828ccd423c4140bbb6636
4
+ data.tar.gz: 2ccb823feaf7f0bc50912ecfb9e08c4f8bb44a14dd4e72c88acb60485ad2a2f6
5
+ SHA512:
6
+ metadata.gz: 50d4c8cb1385c63e6a74ab825f71691cbefbbe66c73786462d690750bd64c8d919c63aa5d3c564c8b8d46d10741a82abe754f482e77416b3ff5c7e1f368abd41
7
+ data.tar.gz: 381f7c4302b0a8d0964e4fcd1b3bcf034ca4401bd6fc68e78768ae7a53931a70b36d7ff2486f72464fd6d5598c0b2723ee375ff762b76efb8647776855cb1c4c
@@ -6,8 +6,6 @@ require 'getoptlong'
6
6
  require 'ostruct'
7
7
  require 'pp'
8
8
 
9
- include MysqlBinlog
10
-
11
9
  def usage(exit_code, message = nil)
12
10
  print "Error: #{message}\n\n" unless message.nil?
13
11
 
@@ -15,13 +13,16 @@ def usage(exit_code, message = nil)
15
13
 
16
14
  Usage:
17
15
  To read from a binary log file on disk:
18
- mysql_binlog_dump [options] -f <filename>
16
+ mysql_binlog_dump [options] <filename(s)>
19
17
 
20
18
  --help, -?
21
19
  Show this help.
22
20
 
23
21
  --file, -f <filename>
24
- Read from a binary log file on disk.
22
+ Read from a binary log file on disk (deprecated).
23
+
24
+ --checksum, -c
25
+ Enable CRC32 checksums.
25
26
 
26
27
  --debug, -d
27
28
  Debug reading from the binary log, showing calls into the reader and the
@@ -45,13 +46,16 @@ end
45
46
 
46
47
  @options = OpenStruct.new
47
48
  @options.file = nil
49
+ @options.checksum = nil
48
50
  @options.debug = false
49
51
  @options.tail = false
50
52
  @options.rotate = false
53
+ @options.filenames = []
51
54
 
52
55
  getopt_options = [
53
56
  [ "--help", "-?", GetoptLong::NO_ARGUMENT ],
54
57
  [ "--file", "-f", GetoptLong::REQUIRED_ARGUMENT ],
58
+ [ "--checksum", "-c", GetoptLong::NO_ARGUMENT ],
55
59
  [ "--debug", "-d", GetoptLong::NO_ARGUMENT ],
56
60
  [ "--tail", "-t", GetoptLong::NO_ARGUMENT ],
57
61
  [ "--rotate", "-r", GetoptLong::NO_ARGUMENT ],
@@ -64,7 +68,9 @@ getopt.each do |opt, arg|
64
68
  when "--help"
65
69
  usage 0
66
70
  when "--file"
67
- @options.file = arg
71
+ @options.filenames << arg
72
+ when "--checksum"
73
+ @options.checksum = :crc32
68
74
  when "--debug"
69
75
  @options.debug = true
70
76
  when "--tail"
@@ -74,34 +80,25 @@ getopt.each do |opt, arg|
74
80
  end
75
81
  end
76
82
 
77
- unless @options.file
78
- usage 1, "A file must be provided with --file/-f"
79
- end
83
+ @options.filenames.concat(ARGV)
80
84
 
81
- reader = BinlogFileReader.new(@options.file)
82
- if @options.debug
83
- reader = DebuggingReader.new(reader, :data => true, :calls => true)
85
+ if @options.filenames.empty?
86
+ usage 1, "One or more filenames must be provided"
84
87
  end
85
- binlog = Binlog.new(reader)
86
88
 
87
- if @options.tail
88
- reader.tail = true
89
- else
90
- reader.tail = false
91
- end
89
+ @options.filenames.each do |filename|
90
+ reader = MysqlBinlog::BinlogFileReader.new(filename)
91
+ if @options.debug
92
+ reader = MysqlBinlog::DebuggingReader.new(reader, :data => true, :calls => true)
93
+ end
94
+ binlog = MysqlBinlog::Binlog.new(reader)
95
+ binlog.checksum = @options.checksum
92
96
 
93
- if @options.rotate
94
- binlog.ignore_rotate = false
95
- else
96
- binlog.ignore_rotate = true
97
- end
97
+ reader.tail = @options.tail
98
+ binlog.ignore_rotate = !@options.rotate
98
99
 
99
- binlog.each_event do |event|
100
- puts "%-30s%-20s%20d" % [
101
- event[:type],
102
- event[:filename],
103
- event[:position],
104
- ]
105
- #pp event
106
- #puts
107
- end
100
+ binlog.each_event do |event|
101
+ pp event
102
+ puts
103
+ end
104
+ end
@@ -0,0 +1,241 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'mysql_binlog'
4
+ require 'getoptlong'
5
+ require 'ostruct'
6
+
7
+ def usage(exit_code, message = nil)
8
+ print "Error: #{message}\n\n" unless message.nil?
9
+
10
+ print <<'END_OF_USAGE'
11
+
12
+ Usage:
13
+ To read from a binary log file on disk:
14
+ mysql_binlog_summary [options] <filename(s)>
15
+
16
+ --help, -?
17
+ Show this help.
18
+
19
+ --file, -f <filename>
20
+ Read from a binary log file on disk (deprecated).
21
+
22
+ --checksum, -c
23
+ Enable CRC32 checksums.
24
+
25
+ --tail, -t
26
+ When reading from a file, follow the end of the binary log file instead
27
+ of exiting when reaching the end. Exit with Control-C.
28
+
29
+ --rotate, -r
30
+ When reading from a file, follow the rotate events which may be at the
31
+ end of a file (due to log rotation) so that the stream can be followed
32
+ through multiple files. This is especially useful with --tail.
33
+
34
+ END_OF_USAGE
35
+
36
+ exit exit_code
37
+ end
38
+
39
+ @options = OpenStruct.new
40
+ @options.tail = false
41
+ @options.rotate = false
42
+ @options.checksum = nil
43
+ @options.filenames = []
44
+
45
+ getopt_options = [
46
+ [ "--help", "-?", GetoptLong::NO_ARGUMENT ],
47
+ [ "--file", "-f", GetoptLong::REQUIRED_ARGUMENT ],
48
+ [ "--tail", "-t", GetoptLong::NO_ARGUMENT ],
49
+ [ "--rotate", "-r", GetoptLong::NO_ARGUMENT ],
50
+ [ "--checksum", "-c", GetoptLong::NO_ARGUMENT ],
51
+ ]
52
+
53
+ getopt = GetoptLong.new(*getopt_options)
54
+
55
+ getopt.each do |opt, arg|
56
+ case opt
57
+ when "--help"
58
+ usage 0
59
+ when "--file"
60
+ @options.filenames << arg
61
+ when "--tail"
62
+ @options.tail = true
63
+ when "--rotate"
64
+ @options.rotate = true
65
+ when "--checksum"
66
+ @options.checksum = :crc32
67
+ end
68
+ end
69
+
70
+ @options.filenames.concat(ARGV)
71
+
72
+ if @options.filenames.empty?
73
+ usage 1, "A file must be provided"
74
+ end
75
+
76
+ files = {}
77
+ min_timestamp = nil
78
+ max_timestamp = nil
79
+ events = []
80
+ events_processed = 0
81
+
82
+ @options.filenames.each do |filename|
83
+ reader = MysqlBinlog::BinlogFileReader.new(filename)
84
+ binlog = MysqlBinlog::Binlog.new(reader)
85
+ reader.tail = @options.tail
86
+ binlog.ignore_rotate = !@options.rotate
87
+ binlog.checksum = @options.checksum
88
+
89
+ file_min_timestamp = nil
90
+ file_max_timestamp = nil
91
+ file_events_processed = 0
92
+
93
+ #binlog.filter_event_types = [:query_event]
94
+ #binlog.filter_flags = [0]
95
+ query_pattern = /^(INSERT|UPDATE|DELETE)\s+(?:(?:INTO|FROM)\s+)?[`]?(\S+?)[`]?\s+/i
96
+
97
+ binlog.each_event do |event|
98
+ verb = nil
99
+ table = nil
100
+
101
+ if event[:type] == :query_event
102
+ if match_query = event[:event][:query].match(query_pattern)
103
+ verb = match_query[1].downcase
104
+ table = match_query[2]
105
+ end
106
+ end
107
+
108
+ if MysqlBinlog::ROW_EVENT_TYPES.include? event[:type]
109
+ verb = event[:type].to_s.sub(/_event_v[12]/, '')
110
+ table = event[:event][:table][:table]
111
+ end
112
+
113
+ timestamp = event[:header][:timestamp]
114
+
115
+ file_min_timestamp = [file_min_timestamp || timestamp, timestamp].min
116
+ file_max_timestamp = [file_max_timestamp || timestamp, timestamp].max
117
+
118
+ net_change = 0
119
+ event[:event][:row_image]&.each do |row_image|
120
+ case verb
121
+ when "delete_rows"
122
+ net_change -= row_image[:before][:size]
123
+ when "update_rows"
124
+ net_change += row_image[:after][:size] - row_image[:before][:size]
125
+ when "write_rows"
126
+ net_change += row_image[:after][:size]
127
+ end
128
+ end
129
+
130
+ events << {
131
+ timestamp: timestamp,
132
+ size: event[:header][:payload_length],
133
+ type: event[:type],
134
+ verb: verb,
135
+ table: table,
136
+ net_change: net_change,
137
+ }
138
+
139
+ file_events_processed += 1
140
+ events_processed += 1
141
+
142
+ if (file_events_processed % 1000) == 0
143
+ puts "%-32s %6d MiB %10d %10d" % [
144
+ filename, event[:position]/(1024**2), file_events_processed, events_processed
145
+ ]
146
+ end
147
+ end
148
+
149
+ files[filename] = {
150
+ filename: filename,
151
+ events: file_events_processed,
152
+ min_timestamp: file_min_timestamp,
153
+ max_timestamp: file_max_timestamp,
154
+ }
155
+
156
+ min_timestamp = [min_timestamp || file_min_timestamp, file_min_timestamp].min
157
+ max_timestamp = [max_timestamp || file_max_timestamp, file_max_timestamp].max
158
+ end
159
+ puts "Done."
160
+ puts
161
+
162
+ duration = max_timestamp - min_timestamp
163
+
164
+ puts "File summary:"
165
+ files.each do |filename, file|
166
+ puts " %-32s%10s%26s%26s" % [
167
+ File.basename(filename),
168
+ file[:events],
169
+ Time.at(file[:min_timestamp]).utc,
170
+ Time.at(file[:max_timestamp]).utc,
171
+ ]
172
+ end
173
+ puts
174
+
175
+ puts "Summary:"
176
+ puts " Files: %d" % [files.size]
177
+ puts " Events: %d" % [events_processed]
178
+ puts " Min Time: %s" % [Time.at(min_timestamp).utc]
179
+ puts " Max Time: %s" % [Time.at(max_timestamp).utc]
180
+ puts " Duration: %ds" % [duration]
181
+ puts " Event Rate: %0.2f/s" % [events_processed.to_f / duration.to_f]
182
+ puts
183
+
184
+ events_by_type = Hash.new(0)
185
+ events_by_verb_and_table = {}
186
+ size_by_verb_and_table = {}
187
+ size_by_table = Hash.new(0)
188
+ net_change_by_verb_and_table = {}
189
+ net_change_by_table = Hash.new(0)
190
+ events.each do |event|
191
+ events_by_type[event[:type]] += 1
192
+ if event[:verb]
193
+ events_by_verb_and_table[event[:verb]] ||= Hash.new(0)
194
+ events_by_verb_and_table[event[:verb]][event[:table]] += 1
195
+ size_by_verb_and_table[event[:verb]] ||= Hash.new(0)
196
+ size_by_verb_and_table[event[:verb]][event[:table]] += event[:size]
197
+ size_by_table[event[:table]] += event[:size]
198
+ net_change_by_verb_and_table[event[:verb]] ||= Hash.new(0)
199
+ net_change_by_verb_and_table[event[:verb]][event[:table]] += event[:net_change]
200
+ net_change_by_table[event[:table]] += event[:net_change]
201
+ end
202
+ end
203
+
204
+ puts "Events by type:"
205
+ events_by_type.sort { |a, b| b[1] <=> a[1] }.each do |type, count|
206
+ puts " %-50s%10d%10.2f/s" % [type, count, count.to_f / duration.to_f]
207
+ end
208
+ puts
209
+
210
+ puts "Events by verb and table:"
211
+ events_by_verb_and_table.sort.each do |verb, table_and_count|
212
+ puts "%s\n" % [verb]
213
+ puts " %-50s%10s%14s%14s%14s" % [
214
+ "", "Count", "Rate/s", "Net (KiB/s)", "Size (KiB/s)"
215
+ ]
216
+ table_and_count.sort { |a, b| b[1] <=> a[1] }.each do |table, count|
217
+ puts " %-50s%10d%14s%+14.2f%14.2f" % [
218
+ table, count, "%10.2f/s" % [count.to_f / duration.to_f],
219
+ net_change_by_verb_and_table[verb][table] / 1024.0 / duration.to_f,
220
+ size_by_verb_and_table[verb][table] / 1024.0 / duration.to_f,
221
+ ]
222
+ end
223
+ puts
224
+ end
225
+
226
+ puts "Event payload by table (top 10):"
227
+ size_by_table.sort { |a, b| b[1].abs <=> a[1].abs }.first(10).each do |table, size|
228
+ puts " %-50s%+10.2f KiB/s" % [
229
+ table, size.to_f / 1024.0 / duration.to_f
230
+ ]
231
+ end
232
+ puts
233
+
234
+ puts "Net change by table (top 10):"
235
+ net_change_by_table.sort { |a, b| b[1].abs <=> a[1].abs }.first(10).each do |table, net_change|
236
+ puts " %-50s%+10.2f KiB/s" % [
237
+ table, net_change.to_f / 1024.0 / duration.to_f
238
+ ]
239
+ end
240
+ puts
241
+
@@ -51,6 +51,7 @@ module MysqlBinlog
51
51
  attr_accessor :filter_flags
52
52
  attr_accessor :ignore_rotate
53
53
  attr_accessor :max_query_length
54
+ attr_accessor :checksum
54
55
 
55
56
  def initialize(reader)
56
57
  @reader = reader
@@ -61,6 +62,7 @@ module MysqlBinlog
61
62
  @filter_flags = nil
62
63
  @ignore_rotate = false
63
64
  @max_query_length = 1048576
65
+ @checksum = :nil
64
66
  end
65
67
 
66
68
  # Rewind to the beginning of the log, if supported by the reader. The
@@ -81,10 +83,16 @@ module MysqlBinlog
81
83
  def read_event_fields(header)
82
84
  # Delegate the parsing of the event content to a method of the same name
83
85
  # in BinlogEventParser.
84
- if event_parser.methods.include? header[:event_type].to_s
86
+ if event_parser.methods.map(&:to_sym).include? header[:event_type]
85
87
  fields = event_parser.send(header[:event_type], header)
86
88
  end
87
89
 
90
+ unless fields
91
+ fields = {
92
+ payload: reader.read(header[:payload_length]),
93
+ }
94
+ end
95
+
88
96
  # Check if we've read past the end of the event. This is normally because
89
97
  # of an unsupported substructure in the event causing field misalignment
90
98
  # or a bug in the event reader method in BinlogEventParser. This may also
@@ -103,6 +111,19 @@ module MysqlBinlog
103
111
  end
104
112
  private :read_event_fields
105
113
 
114
+ def checksum_length
115
+ case @checksum
116
+ when :crc32
117
+ 4
118
+ else
119
+ 0
120
+ end
121
+ end
122
+
123
+ def payload_length(header)
124
+ @fde ? (header[:event_length] - @fde[:header_length] - checksum_length) : 0
125
+ end
126
+
106
127
  # Scan events until finding one that isn't rejected by the filter rules.
107
128
  # If there are no filter rules, this will return the next event provided
108
129
  # by the reader.
@@ -119,6 +140,18 @@ module MysqlBinlog
119
140
  return nil
120
141
  end
121
142
 
143
+ # Skip the remaining part of the header which might not have been
144
+ # parsed.
145
+ if @fde
146
+ reader.seek(position + @fde[:header_length])
147
+ header[:payload_length] = payload_length(header)
148
+ header[:payload_end] = position + @fde[:header_length] + payload_length(header)
149
+ else
150
+ header[:payload_length] = 0
151
+ header[:payload_end] = header[:next_position]
152
+ end
153
+
154
+
122
155
  if @filter_event_types
123
156
  unless @filter_event_types.include? header[:event_type]
124
157
  skip_this_event = true
@@ -26,11 +26,23 @@ module MysqlBinlog
26
26
  :pre_ga_write_rows_event => 20, # (deprecated)
27
27
  :pre_ga_update_rows_event => 21, # (deprecated)
28
28
  :pre_ga_delete_rows_event => 22, # (deprecated)
29
- :write_rows_event => 23, #
30
- :update_rows_event => 24, #
31
- :delete_rows_event => 25, #
29
+ :write_rows_event_v1 => 23, #
30
+ :update_rows_event_v1 => 24, #
31
+ :delete_rows_event_v1 => 25, #
32
32
  :incident_event => 26, #
33
33
  :heartbeat_log_event => 27, #
34
+ :ignorable_log_event => 28, #
35
+ :rows_query_log_event => 29, #
36
+ :write_rows_event_v2 => 30, #
37
+ :update_rows_event_v2 => 31, #
38
+ :delete_rows_event_v2 => 32, #
39
+ :gtid_log_event => 33, #
40
+ :anonymous_gtid_log_event => 34, #
41
+ :previous_gtids_log_event => 35, #
42
+ :transaction_context_event => 36, #
43
+ :view_change_event => 37, #
44
+ :xa_prepare_log_event => 38, #
45
+
34
46
  :table_metadata_event => 50, # Only in Twitter MySQL
35
47
  }
36
48
 
@@ -44,9 +56,12 @@ module MysqlBinlog
44
56
  # have an identical structure, this list can be used by other programs to
45
57
  # know which events can be treated as row events.
46
58
  ROW_EVENT_TYPES = [
47
- :write_rows_event,
48
- :update_rows_event,
49
- :delete_rows_event,
59
+ :write_rows_event_v1,
60
+ :update_rows_event_v1,
61
+ :delete_rows_event_v1,
62
+ :write_rows_event_v2,
63
+ :update_rows_event_v2,
64
+ :delete_rows_event_v2,
50
65
  ]
51
66
 
52
67
  # Values for the +flags+ field that may appear in binary logs. There are
@@ -55,11 +70,14 @@ module MysqlBinlog
55
70
  #
56
71
  # Defined in sql/log_event.h line ~448
57
72
  EVENT_HEADER_FLAGS = {
58
- :binlog_in_use => 0x01, # LOG_EVENT_BINLOG_IN_USE_F
59
- :thread_specific => 0x04, # LOG_EVENT_THREAD_SPECIFIC_F
60
- :suppress_use => 0x08, # LOG_EVENT_SUPPRESS_USE_F
61
- :artificial => 0x20, # LOG_EVENT_ARTIFICIAL_F
62
- :relay_log => 0x40, # LOG_EVENT_RELAY_LOG_F
73
+ :binlog_in_use => 0x0001, # LOG_EVENT_BINLOG_IN_USE_F
74
+ :thread_specific => 0x0004, # LOG_EVENT_THREAD_SPECIFIC_F
75
+ :suppress_use => 0x0008, # LOG_EVENT_SUPPRESS_USE_F
76
+ :artificial => 0x0020, # LOG_EVENT_ARTIFICIAL_F
77
+ :relay_log => 0x0040, # LOG_EVENT_RELAY_LOG_F
78
+ :ignorable => 0x0080, # LOG_EVENT_IGNORABLE_F
79
+ :no_filter => 0x0100, # LOG_EVENT_NO_FILTER_F
80
+ :mts_isolate => 0x0200, # LOG_EVENT_MTS_ISOLATE_F
63
81
  }
64
82
 
65
83
  # A mapping array for all values that may appear in the +status+ field of
@@ -79,8 +97,15 @@ module MysqlBinlog
79
97
  :table_map_for_update, # 9 (Q_TABLE_MAP_FOR_UPDATE_CODE)
80
98
  :master_data_written, # 10 (Q_MASTER_DATA_WRITTEN_CODE)
81
99
  :invoker, # 11 (Q_INVOKER)
100
+ :updated_db_names, # 12 (Q_UPDATED_DB_NAMES)
101
+ :microseconds, # 13 (Q_MICROSECONDS)
102
+ :commit_ts, # 14 (Q_COMMIT_TS)
103
+ :commit_ts2, # 15
104
+ :explicit_defaults_for_timestamp, # 16
82
105
  ]
83
106
 
107
+ QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS = 254
108
+
84
109
  # A mapping hash for all values that may appear in the +flags2+ field of
85
110
  # a query_event.
86
111
  #
@@ -152,6 +177,10 @@ module MysqlBinlog
152
177
  :complete_rows => 1 << 3, # COMPLETE_ROWS_F
153
178
  }
154
179
 
180
+ GENERIC_ROWS_EVENT_VH_FIELD_TYPES = [
181
+ :extra_rows_info, # ROWS_V_EXTRAINFO_TAG
182
+ ]
183
+
155
184
  # Parse binary log events from a provided binary log. Must be driven
156
185
  # externally, but handles all the details of parsing an event header
157
186
  # and the content of the various event types.
@@ -181,11 +210,13 @@ module MysqlBinlog
181
210
  def event_header
182
211
  header = {}
183
212
  header[:timestamp] = parser.read_uint32
184
- header[:event_type] = EVENT_TYPES[parser.read_uint8]
213
+ event_type = parser.read_uint8
214
+ header[:event_type] = EVENT_TYPES[event_type] || "unknown_#{event_type}".to_sym
185
215
  header[:server_id] = parser.read_uint32
186
216
  header[:event_length] = parser.read_uint32
187
217
  header[:next_position] = parser.read_uint32
188
218
  header[:flags] = parser.read_uint_bitmap_by_size_and_name(2, EVENT_HEADER_FLAGS)
219
+
189
220
  header
190
221
  end
191
222
 
@@ -212,6 +243,25 @@ module MysqlBinlog
212
243
  fields
213
244
  end
214
245
 
246
+ def _query_event_status_updated_db_names
247
+ db_count = parser.read_uint8
248
+ return nil if db_count == QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS
249
+
250
+ db_names = []
251
+ db_count.times do |n|
252
+ db_name = ""
253
+ loop do
254
+ c = reader.read(1)
255
+ break if c == "\0"
256
+ db_name << c
257
+ end
258
+ db_names << db_name
259
+ end
260
+
261
+ db_names
262
+ end
263
+ private :_query_event_status_updated_db_names
264
+
215
265
  # Parse a dynamic +status+ structure within a query_event, which consists
216
266
  # of a status_length (uint16) followed by a number of status variables
217
267
  # (determined by the +status_length+) each of which consist of:
@@ -223,7 +273,8 @@ module MysqlBinlog
223
273
  status_length = parser.read_uint16
224
274
  end_position = reader.position + status_length
225
275
  while reader.position < end_position
226
- status_type = QUERY_EVENT_STATUS_TYPES[parser.read_uint8]
276
+ status_type_id = parser.read_uint8
277
+ status_type = QUERY_EVENT_STATUS_TYPES[status_type_id]
227
278
  status[status_type] = case status_type
228
279
  when :flags2
229
280
  parser.read_uint_bitmap_by_size_and_name(4, QUERY_EVENT_FLAGS2)
@@ -252,6 +303,14 @@ module MysqlBinlog
252
303
  parser.read_uint16
253
304
  when :table_map_for_update
254
305
  parser.read_uint64
306
+ when :updated_db_names
307
+ _query_event_status_updated_db_names
308
+ when :commit_ts
309
+ parser.read_uint64
310
+ when :microseconds
311
+ parser.read_uint24
312
+ else
313
+ raise "Unknown status type #{status_type_id}"
255
314
  end
256
315
  end
257
316
 
@@ -334,7 +393,7 @@ module MysqlBinlog
334
393
  :precision => parser.read_uint8,
335
394
  :decimals => parser.read_uint8,
336
395
  }
337
- when :blob, :geometry
396
+ when :blob, :geometry, :json
338
397
  { :length_size => parser.read_uint8 }
339
398
  when :string, :var_string
340
399
  # The :string type sets a :real_type field to indicate the actual type
@@ -350,6 +409,10 @@ module MysqlBinlog
350
409
  else
351
410
  { :max_length => (((metadata >> 4) & 0x300) ^ 0x300) + (metadata & 0x00ff) }
352
411
  end
412
+ when :timestamp2, :datetime2, :time2
413
+ {
414
+ :decimals => parser.read_uint8,
415
+ }
353
416
  end
354
417
  end
355
418
  private :_table_map_event_column_metadata_read
@@ -376,7 +439,7 @@ module MysqlBinlog
376
439
  map_entry[:db] = parser.read_lpstringz
377
440
  map_entry[:table] = parser.read_lpstringz
378
441
  columns = parser.read_varint
379
- columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] }
442
+ columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] || "unknown_#{c}".to_sym }
380
443
  columns_metadata = _table_map_event_column_metadata(columns_type)
381
444
  columns_nullable = parser.read_bit_array(columns)
382
445
 
@@ -415,8 +478,9 @@ module MysqlBinlog
415
478
  fields[:flags] = parser.read_uint16
416
479
  fields[:columns] = columns.times.map do |c|
417
480
  descriptor_length = parser.read_uint32
481
+ column_type = parser.read_uint8
418
482
  @table_map[table_id][:columns][c][:description] = {
419
- :type => MYSQL_TYPES[parser.read_uint8],
483
+ :type => MYSQL_TYPES[column_type] || "unknown_#{column_type}".to_sym,
420
484
  :length => parser.read_uint32,
421
485
  :scale => parser.read_uint8,
422
486
  :character_set => COLLATION[parser.read_uint16],
@@ -435,23 +499,43 @@ module MysqlBinlog
435
499
  # and false values identifies which columns are present.
436
500
  def _generic_rows_event_row_image(header, fields, columns_used)
437
501
  row_image = []
502
+ start_position = reader.position
438
503
  columns_null = parser.read_bit_array(fields[:table][:columns].size)
439
504
  fields[:table][:columns].each_with_index do |column, column_index|
505
+ #puts "column #{column_index} #{column}: used=#{columns_used[column_index]}, null=#{columns_null[column_index]}"
440
506
  if !columns_used[column_index]
441
507
  row_image << nil
442
508
  elsif columns_null[column_index]
443
509
  row_image << { column_index => nil }
444
510
  else
511
+ value = parser.read_mysql_type(column[:type], column[:metadata])
445
512
  row_image << {
446
- column_index =>
447
- parser.read_mysql_type(column[:type], column[:metadata])
513
+ column_index => value,
448
514
  }
449
515
  end
450
516
  end
451
- row_image
517
+ end_position = reader.position
518
+
519
+ {
520
+ image: row_image,
521
+ size: end_position-start_position
522
+ }
452
523
  end
453
524
  private :_generic_rows_event_row_image
454
525
 
526
+ def diff_row_images(before, after)
527
+ diff = {}
528
+ before.each_with_index do |before_column, index|
529
+ after_column = after[index]
530
+ before_value = before_column.first[1]
531
+ after_value = after_column.first[1]
532
+ if before_value != after_value
533
+ diff[index] = { before: before_value, after: after_value }
534
+ end
535
+ end
536
+ diff
537
+ end
538
+
455
539
  # Parse the row images present in a row-based replication row event. This
456
540
  # is rather incomplete right now due missing support for many MySQL types,
457
541
  # but can parse some basic events.
@@ -461,13 +545,14 @@ module MysqlBinlog
461
545
  while reader.position < end_position
462
546
  row_image = {}
463
547
  case header[:event_type]
464
- when :write_rows_event
548
+ when :write_rows_event_v1, :write_rows_event_v2
465
549
  row_image[:after] = _generic_rows_event_row_image(header, fields, columns_used[:after])
466
- when :delete_rows_event
550
+ when :delete_rows_event_v1, :delete_rows_event_v1
467
551
  row_image[:before] = _generic_rows_event_row_image(header, fields, columns_used[:before])
468
- when :update_rows_event
552
+ when :update_rows_event_v1, :update_rows_event_v2
469
553
  row_image[:before] = _generic_rows_event_row_image(header, fields, columns_used[:before])
470
554
  row_image[:after] = _generic_rows_event_row_image(header, fields, columns_used[:after])
555
+ row_image[:diff] = diff_row_images(row_image[:before][:image], row_image[:after][:image])
471
556
  end
472
557
  row_images << row_image
473
558
  end
@@ -483,6 +568,16 @@ module MysqlBinlog
483
568
  end
484
569
  private :_generic_rows_event_row_images
485
570
 
571
+ # Parse the variable header from a v2 rows event. This is only used for
572
+ # ROWS_V_EXTRAINFO_TAG which is used by NDB. Ensure it can be skipped
573
+ # properly but don't bother parsing it.
574
+ def _generic_rows_event_vh
575
+ vh_payload_len = parser.read_uint16 - 2
576
+ return unless vh_payload_len > 0
577
+
578
+ reader.read(vh_payload_len)
579
+ end
580
+
486
581
  # Parse fields for any of the row-based replication row events:
487
582
  # * +Write_rows+ which is used for +INSERT+.
488
583
  # * +Update_rows+ which is used for +UPDATE+.
@@ -491,29 +586,104 @@ module MysqlBinlog
491
586
  # Implemented in sql/log_event.cc line ~8039
492
587
  # in Rows_log_event::write_data_header
493
588
  # and Rows_log_event::write_data_body
494
- def generic_rows_event(header)
589
+ def _generic_rows_event(header, contains_vh: false)
495
590
  fields = {}
496
591
  table_id = parser.read_uint48
497
592
  fields[:table] = @table_map[table_id]
498
593
  fields[:flags] = parser.read_uint_bitmap_by_size_and_name(2, GENERIC_ROWS_EVENT_FLAGS)
594
+
595
+ # Rows_log_event v2 events contain a variable-sized header. Only NDB
596
+ # uses it right now, so let's just make sure it's skipped properly.
597
+ _generic_rows_event_vh if contains_vh
598
+
499
599
  columns = parser.read_varint
500
600
  columns_used = {}
501
601
  case header[:event_type]
502
- when :write_rows_event
602
+ when :write_rows_event_v1, :write_rows_event_v2
503
603
  columns_used[:after] = parser.read_bit_array(columns)
504
- when :delete_rows_event
604
+ when :delete_rows_event_v1, :delete_rows_event_v2
505
605
  columns_used[:before] = parser.read_bit_array(columns)
506
- when :update_rows_event
606
+ when :update_rows_event_v1, :update_rows_event_v2
507
607
  columns_used[:before] = parser.read_bit_array(columns)
508
608
  columns_used[:after] = parser.read_bit_array(columns)
509
609
  end
510
610
  fields[:row_image] = _generic_rows_event_row_images(header, fields, columns_used)
511
611
  fields
512
612
  end
613
+ private :_generic_rows_event
614
+
615
+ def generic_rows_event_v1(header)
616
+ _generic_rows_event(header, contains_vh: false)
617
+ end
618
+
619
+ alias :write_rows_event_v1 :generic_rows_event_v1
620
+ alias :update_rows_event_v1 :generic_rows_event_v1
621
+ alias :delete_rows_event_v1 :generic_rows_event_v1
513
622
 
514
- alias :write_rows_event :generic_rows_event
515
- alias :update_rows_event :generic_rows_event
516
- alias :delete_rows_event :generic_rows_event
623
+ def generic_rows_event_v2(header)
624
+ _generic_rows_event(header, contains_vh: true)
625
+ end
626
+
627
+ alias :write_rows_event_v2 :generic_rows_event_v2
628
+ alias :update_rows_event_v2 :generic_rows_event_v2
629
+ alias :delete_rows_event_v2 :generic_rows_event_v2
630
+
631
+ def rows_query_log_event(header)
632
+ reader.read(1) # skip useless byte length which is unused
633
+ { query: reader.read(header[:payload_length]-1) }
634
+ end
635
+
636
+ def in_hex(bytes)
637
+ bytes.each_byte.map { |c| "%02x" % c.ord }.join
638
+ end
517
639
 
640
+ def format_gtid_sid(sid)
641
+ [0..3, 4..5, 6..7, 8..9, 10..15].map { |r| in_hex(sid[r]) }.join("-")
642
+ end
643
+
644
+ # 6d9190a2-cca6-11e8-aa8c-42010aef0019:551845019
645
+ def format_gtid(sid, gno_or_ivs)
646
+ "#{format_gtid_sid(sid)}:#{gno_or_ivs}"
647
+ end
648
+
649
+ def previous_gtids_log_event(header)
650
+ n_sids = parser.read_uint64
651
+
652
+ gtids = []
653
+ n_sids.times do
654
+ sid = parser.read_nstring(16)
655
+ n_ivs = parser.read_uint64
656
+ ivs = []
657
+ n_ivs.times do
658
+ iv_start = parser.read_uint64
659
+ iv_end = parser.read_uint64
660
+ ivs << "#{iv_start}-#{iv_end}"
661
+ end
662
+ gtids << format_gtid(sid, ivs.join(":"))
663
+ end
664
+
665
+ {
666
+ previous_gtids: gtids
667
+ }
668
+ end
669
+
670
+ def gtid_log_event(header)
671
+ flags = parser.read_uint8
672
+ sid = parser.read_nstring(16)
673
+ gno = parser.read_uint64
674
+ lts_type = parser.read_uint8
675
+ lts_last_committed = parser.read_uint64
676
+ lts_sequence_number = parser.read_uint64
677
+
678
+ {
679
+ flags: flags,
680
+ gtid: format_gtid(sid, gno),
681
+ lts: {
682
+ type: lts_type,
683
+ last_committed: lts_last_committed,
684
+ sequence_number: lts_sequence_number,
685
+ },
686
+ }
687
+ end
518
688
  end
519
689
  end
@@ -1,3 +1,5 @@
1
+ require 'bigdecimal'
2
+
1
3
  module MysqlBinlog
2
4
  # All MySQL types mapping to their integer values.
3
5
  MYSQL_TYPES_HASH = {
@@ -18,6 +20,10 @@ module MysqlBinlog
18
20
  :newdate => 14,
19
21
  :varchar => 15,
20
22
  :bit => 16,
23
+ :timestamp2 => 17,
24
+ :datetime2 => 18,
25
+ :time2 => 19,
26
+ :json => 245,
21
27
  :newdecimal => 246,
22
28
  :enum => 247,
23
29
  :set => 248,
@@ -58,12 +64,28 @@ module MysqlBinlog
58
64
  reader.read(2).unpack("v").first
59
65
  end
60
66
 
67
+ # Read an unsigned 16-bit (2-byte) big-endian integer.
68
+ def read_uint16_be
69
+ reader.read(2).unpack("n").first
70
+ end
71
+
61
72
  # Read an unsigned 24-bit (3-byte) integer.
62
73
  def read_uint24
63
74
  a, b, c = reader.read(3).unpack("CCC")
64
75
  a + (b << 8) + (c << 16)
65
76
  end
66
77
 
78
+ # Read an unsigned 24-bit (3-byte) big-endian integer.
79
+ def read_uint24_be
80
+ a, b = reader.read(3).unpack("nC")
81
+ (a << 8) + b
82
+ end
83
+
84
+ # Read an unsigned 32-bit (4-byte) integer.
85
+ def read_uint32_be
86
+ reader.read(4).unpack("N").first
87
+ end
88
+
67
89
  # Read an unsigned 32-bit (4-byte) integer.
68
90
  def read_uint32
69
91
  reader.read(4).unpack("V").first
@@ -75,6 +97,12 @@ module MysqlBinlog
75
97
  a + (b << 8)
76
98
  end
77
99
 
100
+ # Read an unsigned 40-bit (5-byte) big-endian integer.
101
+ def read_uint40_be
102
+ a, b = reader.read(5).unpack("NC")
103
+ (a << 8) + b
104
+ end
105
+
78
106
  # Read an unsigned 48-bit (6-byte) integer.
79
107
  def read_uint48
80
108
  a, b, c = reader.read(6).unpack("vvv")
@@ -89,7 +117,12 @@ module MysqlBinlog
89
117
 
90
118
  # Read an unsigned 64-bit (8-byte) integer.
91
119
  def read_uint64
92
- reader.read(8).unpack("Q").first
120
+ reader.read(8).unpack("Q<").first
121
+ end
122
+
123
+ # Read an unsigned 64-bit (8-byte) integer.
124
+ def read_uint64_be
125
+ reader.read(8).unpack("Q>").first
93
126
  end
94
127
 
95
128
  # Read a signed 8-bit (1-byte) integer.
@@ -99,13 +132,13 @@ module MysqlBinlog
99
132
 
100
133
  # Read a signed 16-bit (2-byte) big-endian integer.
101
134
  def read_int16_be
102
- reader.read(2).reverse.unpack('s').first
135
+ reader.read(2).unpack('n').first
103
136
  end
104
137
 
105
138
  # Read a signed 24-bit (3-byte) big-endian integer.
106
139
  def read_int24_be
107
140
  a, b, c = reader.read(3).unpack('CCC')
108
- if a & 128
141
+ if (a & 128) == 0
109
142
  (a << 16) | (b << 8) | c
110
143
  else
111
144
  (-1 << 24) | (a << 16) | (b << 8) | c
@@ -114,9 +147,9 @@ module MysqlBinlog
114
147
 
115
148
  # Read a signed 32-bit (4-byte) big-endian integer.
116
149
  def read_int32_be
117
- reader.read(4).reverse.unpack('l').first
150
+ reader.read(4).unpack('N').first
118
151
  end
119
-
152
+
120
153
  def read_uint_by_size(size)
121
154
  case size
122
155
  when 1
@@ -272,7 +305,7 @@ module MysqlBinlog
272
305
  str << value.to_s
273
306
  end
274
307
 
275
- BigDecimal.new(str)
308
+ BigDecimal(str)
276
309
  end
277
310
 
278
311
  # Read an array of unsigned 8-bit (1-byte) integers.
@@ -359,6 +392,47 @@ module MysqlBinlog
359
392
  ]
360
393
  end
361
394
 
395
+ def convert_mysql_type_datetimef(int_part, frac_part)
396
+ year_month = extract_bits(int_part, 17, 22)
397
+ year = year_month / 13
398
+ month = year_month % 13
399
+ day = extract_bits(int_part, 5, 17)
400
+ hour = extract_bits(int_part, 5, 12)
401
+ minute = extract_bits(int_part, 6, 6)
402
+ second = extract_bits(int_part, 6, 0)
403
+
404
+ "%04i-%02i-%02i %02i:%02i:%02i.%06i" % [
405
+ year,
406
+ month,
407
+ day,
408
+ hour,
409
+ minute,
410
+ second,
411
+ frac_part,
412
+ ]
413
+ end
414
+
415
+ def read_frac_part(decimals)
416
+ case decimals
417
+ when 0
418
+ 0
419
+ when 1, 2
420
+ read_uint8 * 10000
421
+ when 3, 4
422
+ read_uint16_be * 100
423
+ when 5, 6
424
+ read_uint24_be
425
+ end
426
+ end
427
+
428
+ def read_datetimef(decimals)
429
+ convert_mysql_type_datetimef(read_uint40_be, read_frac_part(decimals))
430
+ end
431
+
432
+ def read_timestamp2(decimals)
433
+ read_uint32_be + (read_frac_part(decimals) / 1000000)
434
+ end
435
+
362
436
  # Read a single field, provided the MySQL column type as a symbol. Not all
363
437
  # types are currently supported.
364
438
  def read_mysql_type(type, metadata=nil)
@@ -382,10 +456,12 @@ module MysqlBinlog
382
456
  when :varchar, :string
383
457
  prefix_size = (metadata[:max_length] > 255) ? 2 : 1
384
458
  read_lpstring(prefix_size)
385
- when :blob, :geometry
459
+ when :blob, :geometry, :json
386
460
  read_lpstring(metadata[:length_size])
387
461
  when :timestamp
388
462
  read_uint32
463
+ when :timestamp2
464
+ read_timestamp2(metadata[:decimals])
389
465
  when :year
390
466
  read_uint8 + 1900
391
467
  when :date
@@ -394,6 +470,8 @@ module MysqlBinlog
394
470
  convert_mysql_type_time(read_uint24)
395
471
  when :datetime
396
472
  convert_mysql_type_datetime(read_uint64)
473
+ when :datetime2
474
+ read_datetimef(metadata[:decimals])
397
475
  when :enum, :set
398
476
  read_uint_by_size(metadata[:size])
399
477
  when :bit
@@ -20,7 +20,7 @@ module MysqlBinlog
20
20
  def open_file(filename)
21
21
  @dirname = File.dirname(filename)
22
22
  @filename = File.basename(filename)
23
- @binlog = File.open(filename, mode="r")
23
+ @binlog = File.open(filename, "r:BINARY")
24
24
 
25
25
  verify_magic
26
26
  end
@@ -69,7 +69,7 @@ module MysqlBinlog
69
69
  end
70
70
 
71
71
  def remaining(header)
72
- header[:next_position] - @binlog.tell
72
+ header[:payload_end] - @binlog.tell
73
73
  end
74
74
 
75
75
  def skip(header)
@@ -1,3 +1,3 @@
1
1
  module MysqlBinlog
2
- VERSION = "0.3.2"
2
+ VERSION = "0.3.7"
3
3
  end
metadata CHANGED
@@ -1,32 +1,25 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: mysql_binlog
3
- version: !ruby/object:Gem::Version
4
- hash: 23
5
- prerelease:
6
- segments:
7
- - 0
8
- - 3
9
- - 2
10
- version: 0.3.2
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.7
11
5
  platform: ruby
12
- authors:
6
+ authors:
13
7
  - Jeremy Cole
14
8
  autorequire:
15
9
  bindir: bin
16
10
  cert_chain: []
17
-
18
- date: 2012-11-07 00:00:00 Z
11
+ date: 2021-04-14 00:00:00.000000000 Z
19
12
  dependencies: []
20
-
21
13
  description: Library for parsing MySQL binary logs in Ruby
22
14
  email: jeremy@jcole.us
23
- executables:
15
+ executables:
24
16
  - mysql_binlog_dump
17
+ - mysql_binlog_summary
25
18
  extensions: []
26
-
27
19
  extra_rdoc_files: []
28
-
29
- files:
20
+ files:
21
+ - bin/mysql_binlog_dump
22
+ - bin/mysql_binlog_summary
30
23
  - lib/mysql_binlog.rb
31
24
  - lib/mysql_binlog/binlog.rb
32
25
  - lib/mysql_binlog/binlog_event_parser.rb
@@ -36,40 +29,27 @@ files:
36
29
  - lib/mysql_binlog/reader/binlog_stream_reader.rb
37
30
  - lib/mysql_binlog/reader/debugging_reader.rb
38
31
  - lib/mysql_binlog/version.rb
39
- - bin/mysql_binlog_dump
40
32
  homepage: http://jcole.us/
41
- licenses: []
42
-
33
+ licenses:
34
+ - BSD-3-Clause
35
+ metadata: {}
43
36
  post_install_message:
44
37
  rdoc_options: []
45
-
46
- require_paths:
38
+ require_paths:
47
39
  - lib
48
- required_ruby_version: !ruby/object:Gem::Requirement
49
- none: false
50
- requirements:
40
+ required_ruby_version: !ruby/object:Gem::Requirement
41
+ requirements:
51
42
  - - ">="
52
- - !ruby/object:Gem::Version
53
- hash: 3
54
- segments:
55
- - 0
56
- version: "0"
57
- required_rubygems_version: !ruby/object:Gem::Requirement
58
- none: false
59
- requirements:
43
+ - !ruby/object:Gem::Version
44
+ version: '0'
45
+ required_rubygems_version: !ruby/object:Gem::Requirement
46
+ requirements:
60
47
  - - ">="
61
- - !ruby/object:Gem::Version
62
- hash: 3
63
- segments:
64
- - 0
65
- version: "0"
48
+ - !ruby/object:Gem::Version
49
+ version: '0'
66
50
  requirements: []
67
-
68
- rubyforge_project:
69
- rubygems_version: 1.8.10
51
+ rubygems_version: 3.1.4
70
52
  signing_key:
71
- specification_version: 3
53
+ specification_version: 4
72
54
  summary: MySQL Binary Log Parser
73
55
  test_files: []
74
-
75
- has_rdoc: