mysql_binlog 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,16 +1,100 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require 'pp'
4
3
  require 'mysql_binlog'
4
+ require 'bigdecimal'
5
+ require 'getoptlong'
6
+ require 'ostruct'
7
+ require 'pp'
5
8
 
6
9
  include MysqlBinlog
7
10
 
8
- reader = BinlogFileReader.new(ARGV.first)
9
- #reader = DebuggingReader.new(reader, :data => true, :calls => true)
11
+ def usage(exit_code, message = nil)
12
+ print "Error: #{message}\n\n" unless message.nil?
13
+
14
+ print <<'END_OF_USAGE'
15
+
16
+ Usage:
17
+ To read from a binary log file on disk:
18
+ mysql_binlog_dump [options] -f <filename>
19
+
20
+ --help, -?
21
+ Show this help.
22
+
23
+ --file, -f <filename>
24
+ Read from a binary log file on disk.
25
+
26
+ --debug, -d
27
+ Debug reading from the binary log, showing calls into the reader and the
28
+ data bytes read. This is useful for debugging the mysql_binlog library
29
+ as well as debugging problems with binary logs.
30
+
31
+ --tail, -t
32
+ When reading from a file, follow the end of the binary log file instead
33
+ of exiting when reaching the end. Exit with Control-C.
34
+
35
+ --rotate, -r
36
+ When reading from a file, follow the rotate events which may be at the
37
+ end of a file (due to log rotation) so that the stream can be followed
38
+ through multiple files. This is especially useful with --tail.
39
+
40
+ END_OF_USAGE
41
+
42
+ exit exit_code
43
+ end
44
+
45
+
46
+ @options = OpenStruct.new
47
+ @options.file = nil
48
+ @options.debug = false
49
+ @options.tail = false
50
+ @options.rotate = false
51
+
52
+ getopt_options = [
53
+ [ "--help", "-?", GetoptLong::NO_ARGUMENT ],
54
+ [ "--file", "-f", GetoptLong::REQUIRED_ARGUMENT ],
55
+ [ "--debug", "-d", GetoptLong::NO_ARGUMENT ],
56
+ [ "--tail", "-t", GetoptLong::NO_ARGUMENT ],
57
+ [ "--rotate", "-r", GetoptLong::NO_ARGUMENT ],
58
+ ]
59
+
60
+ getopt = GetoptLong.new(*getopt_options)
61
+
62
+ getopt.each do |opt, arg|
63
+ case opt
64
+ when "--help"
65
+ usage 0
66
+ when "--file"
67
+ @options.file = arg
68
+ when "--debug"
69
+ @options.debug = true
70
+ when "--tail"
71
+ @options.tail = true
72
+ when "--rotate"
73
+ @options.rotate = true
74
+ end
75
+ end
76
+
77
+ unless @options.file
78
+ usage 1, "A file must be provided with --file/-f"
79
+ end
80
+
81
+ reader = BinlogFileReader.new(@options.file)
82
+ if @options.debug
83
+ reader = DebuggingReader.new(reader, :data => true, :calls => true)
84
+ end
10
85
  binlog = Binlog.new(reader)
11
86
 
12
- #reader.tail = true
13
- #binlog.ignore_rotate = true
87
+ if @options.tail
88
+ reader.tail = true
89
+ else
90
+ reader.tail = false
91
+ end
92
+
93
+ if @options.rotate
94
+ binlog.ignore_rotate = false
95
+ else
96
+ binlog.ignore_rotate = true
97
+ end
14
98
 
15
99
  binlog.each_event do |event|
16
100
  pp event
@@ -2,6 +2,9 @@ module MysqlBinlog
2
2
  # This version of the binary log format is not supported by this library.
3
3
  class UnsupportedVersionException < Exception; end
4
4
 
5
+ # This field type is not supported by this library.
6
+ class UnsupportedTypeException < Exception; end
7
+
5
8
  # An error was encountered when trying to read the log, which was likely
6
9
  # due to garbage data in the log. Continuing is likely impossible.
7
10
  class MalformedBinlogException < Exception; end
@@ -141,8 +144,9 @@ module MysqlBinlog
141
144
 
142
145
  case header[:event_type]
143
146
  when :rotate_event
144
- next if ignore_rotate
145
- reader.rotate(fields[:name], fields[:pos])
147
+ unless ignore_rotate
148
+ reader.rotate(fields[:name], fields[:pos])
149
+ end
146
150
  when :format_description_event
147
151
  process_fde(fields)
148
152
  end
@@ -1,36 +1,52 @@
1
1
  module MysqlBinlog
2
- # An array to quickly map an integer event type to its symbol.
2
+ # A hash of all possible event type IDs.
3
3
  #
4
4
  # Enumerated in sql/log_event.h line ~539 as Log_event_type
5
- EVENT_TYPES = [
6
- :unknown_event, # 0
7
- :start_event_v3, # 1 (deprecated)
8
- :query_event, # 2
9
- :stop_event, # 3
10
- :rotate_event, # 4
11
- :intvar_event, # 5
12
- :load_event, # 6 (deprecated)
13
- :slave_event, # 7 (deprecated)
14
- :create_file_event, # 8 (deprecated)
15
- :append_block_event, # 9
16
- :exec_load_event, # 10 (deprecated)
17
- :delete_file_event, # 11
18
- :new_load_event, # 12 (deprecated)
19
- :rand_event, # 13
20
- :user_var_event, # 14
21
- :format_description_event, # 15
22
- :xid_event, # 16
23
- :begin_load_query_event, # 17
24
- :execute_load_query_event, # 18
25
- :table_map_event, # 19
26
- :pre_ga_write_rows_event, # 20 (deprecated)
27
- :pre_ga_update_rows_event, # 21 (deprecated)
28
- :pre_ga_delete_rows_event, # 22 (deprecated)
29
- :write_rows_event, # 23
30
- :update_rows_event, # 24
31
- :delete_rows_event, # 25
32
- :incident_event, # 26
33
- :heartbeat_log_event, # 27
5
+ EVENT_TYPES_HASH = {
6
+ :unknown_event => 0, #
7
+ :start_event_v3 => 1, # (deprecated)
8
+ :query_event => 2, #
9
+ :stop_event => 3, #
10
+ :rotate_event => 4, #
11
+ :intvar_event => 5, #
12
+ :load_event => 6, # (deprecated)
13
+ :slave_event => 7, # (deprecated)
14
+ :create_file_event => 8, # (deprecated)
15
+ :append_block_event => 9, #
16
+ :exec_load_event => 10, # (deprecated)
17
+ :delete_file_event => 11, #
18
+ :new_load_event => 12, # (deprecated)
19
+ :rand_event => 13, #
20
+ :user_var_event => 14, #
21
+ :format_description_event => 15, #
22
+ :xid_event => 16, #
23
+ :begin_load_query_event => 17, #
24
+ :execute_load_query_event => 18, #
25
+ :table_map_event => 19, #
26
+ :pre_ga_write_rows_event => 20, # (deprecated)
27
+ :pre_ga_update_rows_event => 21, # (deprecated)
28
+ :pre_ga_delete_rows_event => 22, # (deprecated)
29
+ :write_rows_event => 23, #
30
+ :update_rows_event => 24, #
31
+ :delete_rows_event => 25, #
32
+ :incident_event => 26, #
33
+ :heartbeat_log_event => 27, #
34
+ :table_metadata_event => 50, # Only in Twitter MySQL
35
+ }
36
+
37
+ # A lookup array to map an integer event type ID to its symbol.
38
+ EVENT_TYPES = EVENT_TYPES_HASH.inject(Array.new(256)) do |type_array, item|
39
+ type_array[item[1]] = item[0]
40
+ type_array
41
+ end
42
+
43
+ # A list of supported row-based replication event types. Since these all
44
+ # have an identical structure, this list can be used by other programs to
45
+ # know which events can be treated as row events.
46
+ ROW_EVENT_TYPES = [
47
+ :write_rows_event,
48
+ :update_rows_event,
49
+ :delete_rows_event,
34
50
  ]
35
51
 
36
52
  # Values for the +flags+ field that may appear in binary logs. There are
@@ -96,6 +112,35 @@ module MysqlBinlog
96
112
  :bit_len_exact => 1 << 0, # TM_BIT_LEN_EXACT_F
97
113
  }
98
114
 
115
+ # A mapping array for all values that may appear in the +flags+ field of a
116
+ # table_metadata_event.
117
+ #
118
+ # There are none of these at the moment.
119
+ TABLE_METADATA_EVENT_FLAGS = {
120
+ }
121
+
122
+ # A mapping hash for all values that may appear in the +flags+ field of
123
+ # a column descriptor for a table_metadata_event.
124
+ #
125
+ # Defined in include/mysql_com.h line ~92
126
+ TABLE_METADATA_EVENT_COLUMN_FLAGS = {
127
+ :not_null => 1, # NOT_NULL_FLAG
128
+ :primary_key => 2, # PRI_KEY_FLAG
129
+ :unique_key => 4, # UNIQUE_KEY_FLAG
130
+ :multiple_key => 8, # MULTIPLE_KEY_FLAG
131
+ :blob => 16, # BLOB_FLAG
132
+ :unsigned => 32, # UNSIGNED_FLAG
133
+ :zerofill => 64, # ZEROFILL_FLAG
134
+ :binary => 128, # BINARY_FLAG
135
+ :enum => 256, # ENUM_FLAG
136
+ :auto_increment => 512, # AUTO_INCREMENT_FLAG
137
+ :timestamp => 1024, # TIMESTAMP_FLAG
138
+ :set => 2048, # SET_FLAG
139
+ :no_default_value => 4096, # NO_DEFAULT_VALUE_FLAG
140
+ :on_update_now => 8192, # ON_UPDATE_NOW_FLAG
141
+ :part_key => 16384, # PART_KEY_FLAG
142
+ }
143
+
99
144
  # A mapping array for all values that may appear in the +flags+ field of a
100
145
  # write_rows_event, update_rows_event, or delete_rows_event.
101
146
  #
@@ -296,12 +341,14 @@ module MysqlBinlog
296
341
  # which is fundamentally incompatible with :string parsing. Setting
297
342
  # a :type key in this hash will cause table_map_event to override the
298
343
  # main field :type with the provided type here.
299
- real_type = MYSQL_TYPES[parser.read_uint8]
344
+ # See Field_string::do_save_field_metadata for reference.
345
+ metadata = (parser.read_uint8 << 8) + parser.read_uint8
346
+ real_type = MYSQL_TYPES[metadata >> 8]
300
347
  case real_type
301
348
  when :enum, :set
302
- { :type => real_type, :size => parser.read_uint8 }
349
+ { :type => real_type, :size => metadata & 0x00ff }
303
350
  else
304
- { :max_length => parser.read_uint8 }
351
+ { :max_length => (((metadata >> 4) & 0x300) ^ 0x300) + (metadata & 0x00ff) }
305
352
  end
306
353
  end
307
354
  end
@@ -353,6 +400,36 @@ module MysqlBinlog
353
400
  fields
354
401
  end
355
402
 
403
+ # Parse fields for a +Table_metadata+ event, which is specific to
404
+ # Twitter MySQL releases at the moment.
405
+ #
406
+ # Implemented in sql/log_event.cc line ~8772 (in Twitter MySQL)
407
+ # in Table_metadata_log_event::write_data_header
408
+ # and Table_metadata_log_event::write_data_body
409
+ def table_metadata_event(header)
410
+ fields = {}
411
+ table_id = parser.read_uint48
412
+ columns = parser.read_uint16
413
+
414
+ fields[:table] = @table_map[table_id]
415
+ fields[:flags] = parser.read_uint16
416
+ fields[:columns] = columns.times.map do |c|
417
+ descriptor_length = parser.read_uint32
418
+ @table_map[table_id][:columns][c][:description] = {
419
+ :type => MYSQL_TYPES[parser.read_uint8],
420
+ :length => parser.read_uint32,
421
+ :scale => parser.read_uint8,
422
+ :character_set => COLLATION[parser.read_uint16],
423
+ :flags => parser.read_uint_bitmap_by_size_and_name(2,
424
+ TABLE_METADATA_EVENT_COLUMN_FLAGS),
425
+ :name => parser.read_varstring,
426
+ :type_name => parser.read_varstring,
427
+ :comment => parser.read_varstring,
428
+ }
429
+ end
430
+ fields
431
+ end
432
+
356
433
  # Parse a single row image, which is comprised of a series of columns. Not
357
434
  # all columns are present in the row image, the columns_used array of true
358
435
  # and false values identifies which columns are present.
@@ -363,10 +440,11 @@ module MysqlBinlog
363
440
  if !columns_used[column_index]
364
441
  row_image << nil
365
442
  elsif columns_null[column_index]
366
- row_image << { column => nil }
443
+ row_image << { column_index => nil }
367
444
  else
368
445
  row_image << {
369
- column => parser.read_mysql_type(column[:type], column[:metadata])
446
+ column_index =>
447
+ parser.read_mysql_type(column[:type], column[:metadata])
370
448
  }
371
449
  end
372
450
  end
@@ -438,4 +516,4 @@ module MysqlBinlog
438
516
  alias :delete_rows_event :generic_rows_event
439
517
 
440
518
  end
441
- end
519
+ end
@@ -69,17 +69,54 @@ module MysqlBinlog
69
69
  reader.read(4).unpack("V").first
70
70
  end
71
71
 
72
+ # Read an unsigned 40-bit (5-byte) integer.
73
+ def read_uint40
74
+ a, b = reader.read(5).unpack("CV")
75
+ a + (b << 8)
76
+ end
77
+
72
78
  # Read an unsigned 48-bit (6-byte) integer.
73
79
  def read_uint48
74
80
  a, b, c = reader.read(6).unpack("vvv")
75
81
  a + (b << 16) + (c << 32)
76
82
  end
77
83
 
84
+ # Read an unsigned 56-bit (7-byte) integer.
85
+ def read_uint56
86
+ a, b, c = reader.read(7).unpack("CvV")
87
+ a + (b << 8) + (c << 24)
88
+ end
89
+
78
90
  # Read an unsigned 64-bit (8-byte) integer.
79
91
  def read_uint64
80
92
  reader.read(8).unpack("Q").first
81
93
  end
82
94
 
95
+ # Read a signed 8-bit (1-byte) integer.
96
+ def read_int8
97
+ reader.read(1).unpack("c").first
98
+ end
99
+
100
+ # Read a signed 16-bit (2-byte) big-endian integer.
101
+ def read_int16_be
102
+ reader.read(2).reverse.unpack('s').first
103
+ end
104
+
105
+ # Read a signed 24-bit (3-byte) big-endian integer.
106
+ def read_int24_be
107
+ a, b, c = reader.read(3).unpack('CCC')
108
+ if a & 128
109
+ (a << 16) | (b << 8) | c
110
+ else
111
+ (-1 << 24) | (a << 16) | (b << 8) | c
112
+ end
113
+ end
114
+
115
+ # Read a signed 32-bit (4-byte) big-endian integer.
116
+ def read_int32_be
117
+ reader.read(4).reverse.unpack('l').first
118
+ end
119
+
83
120
  def read_uint_by_size(size)
84
121
  case size
85
122
  when 1
@@ -90,13 +127,32 @@ module MysqlBinlog
90
127
  read_uint24
91
128
  when 4
92
129
  read_uint32
130
+ when 5
131
+ read_uint40
93
132
  when 6
94
133
  read_uint48
134
+ when 7
135
+ read_uint56
95
136
  when 8
96
137
  read_uint64
97
138
  end
98
139
  end
99
140
 
141
+ def read_int_be_by_size(size)
142
+ case size
143
+ when 1
144
+ read_int8
145
+ when 2
146
+ read_int16_be
147
+ when 3
148
+ read_int24_be
149
+ when 4
150
+ read_int32_be
151
+ else
152
+ raise "read_int#{size*8}_be not implemented"
153
+ end
154
+ end
155
+
100
156
  # Read a single-precision (4-byte) floating point number.
101
157
  def read_float
102
158
  reader.read(4).unpack("e").first
@@ -168,6 +224,57 @@ module MysqlBinlog
168
224
  read_nstring(length)
169
225
  end
170
226
 
227
+ # Read a (new) decimal value. The value is stored as a sequence of signed
228
+ # big-endian integers, each representing up to 9 digits of the integral
229
+ # and fractional parts. The first integer of the integral part and/or the
230
+ # last integer of the fractional part might be compressed (or packed) and
231
+ # are of variable length. The remaining integers (if any) are
232
+ # uncompressed and 32 bits wide.
233
+ def read_newdecimal(precision, scale)
234
+ digits_per_integer = 9
235
+ compressed_bytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4]
236
+ integral = (precision - scale)
237
+ uncomp_integral = integral / digits_per_integer
238
+ uncomp_fractional = scale / digits_per_integer
239
+ comp_integral = integral - (uncomp_integral * digits_per_integer)
240
+ comp_fractional = scale - (uncomp_fractional * digits_per_integer)
241
+
242
+ # The sign is encoded in the high bit of the first byte/digit. The byte
243
+ # might be part of a larger integer, so apply the optional bit-flipper
244
+ # and push back the byte into the input stream.
245
+ value = read_uint8
246
+ str, mask = (value & 0x80 != 0) ? ["", 0] : ["-", -1]
247
+ reader.unget(value ^ 0x80)
248
+
249
+ size = compressed_bytes[comp_integral]
250
+
251
+ if size > 0
252
+ value = read_int_be_by_size(size) ^ mask
253
+ str << value.to_s
254
+ end
255
+
256
+ (1..uncomp_integral).each do
257
+ value = read_int32_be ^ mask
258
+ str << value.to_s
259
+ end
260
+
261
+ str << "."
262
+
263
+ (1..uncomp_fractional).each do
264
+ value = read_int32_be ^ mask
265
+ str << value.to_s
266
+ end
267
+
268
+ size = compressed_bytes[comp_fractional]
269
+
270
+ if size > 0
271
+ value = read_int_be_by_size(size) ^ mask
272
+ str << value.to_s
273
+ end
274
+
275
+ BigDecimal.new(str)
276
+ end
277
+
171
278
  # Read an array of unsigned 8-bit (1-byte) integers.
172
279
  def read_uint8_array(length)
173
280
  reader.read(length).bytes.to_a
@@ -178,20 +285,36 @@ module MysqlBinlog
178
285
  # events that need bitmaps, as well as for the BIT type.
179
286
  def read_bit_array(length)
180
287
  data = reader.read((length+7)/8)
181
- data.unpack("B*").first.reverse. # Unpack into a string of "10101"
288
+ data.unpack("b*").first. # Unpack into a string of "10101"
182
289
  split("").map { |c| c == "1" }.shift(length) # Return true/false array
183
290
  end
184
291
 
185
292
  # Read a uint value using the provided size, and convert it to an array
186
293
  # of symbols derived from a mapping table provided.
187
- def read_uint_bitmap_by_size_and_name(size, names)
294
+ def read_uint_bitmap_by_size_and_name(size, bit_names)
188
295
  value = read_uint_by_size(size)
189
- names.inject([]) do |result, (name, bit_value)|
296
+ named_bits = []
297
+
298
+ # Do an efficient scan for the named bits we know about using the hash
299
+ # provided.
300
+ bit_names.each do |(name, bit_value)|
190
301
  if (value & bit_value) != 0
191
- result << name
302
+ value -= bit_value
303
+ named_bits << name
192
304
  end
193
- result
194
305
  end
306
+
307
+ # If anything is left over in +value+, add "unknown" names to the result
308
+ # so that they can be identified and corrected.
309
+ if value > 0
310
+ 0.upto(size * 8).map { |n| 1 << n }.each do |bit_value|
311
+ if (value & bit_value) != 0
312
+ named_bits << "unknown_#{bit_value}".to_sym
313
+ end
314
+ end
315
+ end
316
+
317
+ named_bits
195
318
  end
196
319
 
197
320
  # Extract a number of sequential bits at a given offset within an integer.
@@ -254,10 +377,11 @@ module MysqlBinlog
254
377
  read_float
255
378
  when :double
256
379
  read_double
257
- when :string, :var_string
380
+ when :var_string
258
381
  read_varstring
259
- when :varchar
260
- read_lpstring(2)
382
+ when :varchar, :string
383
+ prefix_size = (metadata[:max_length] > 255) ? 2 : 1
384
+ read_lpstring(prefix_size)
261
385
  when :blob, :geometry
262
386
  read_lpstring(metadata[:length_size])
263
387
  when :timestamp
@@ -273,9 +397,15 @@ module MysqlBinlog
273
397
  when :enum, :set
274
398
  read_uint_by_size(metadata[:size])
275
399
  when :bit
276
- read_bit_array(metadata[:bits])
277
- #when :newdecimal
400
+ byte_length = (metadata[:bits]+7)/8
401
+ read_uint_by_size(byte_length)
402
+ when :newdecimal
403
+ precision = metadata[:precision]
404
+ scale = metadata[:decimals]
405
+ read_newdecimal(precision, scale)
406
+ else
407
+ raise UnsupportedTypeException.new("Type #{type} is not supported.")
278
408
  end
279
409
  end
280
410
  end
281
- end
411
+ end
@@ -58,7 +58,11 @@ module MysqlBinlog
58
58
  def seek(pos)
59
59
  @binlog.seek(pos)
60
60
  end
61
-
61
+
62
+ def unget(char)
63
+ @binlog.ungetc(char)
64
+ end
65
+
62
66
  def end?
63
67
  return false if tail
64
68
  @binlog.eof?
@@ -91,4 +95,4 @@ module MysqlBinlog
91
95
  data
92
96
  end
93
97
  end
94
- end
98
+ end
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mysql_binlog
3
3
  version: !ruby/object:Gem::Version
4
- hash: 23
4
+ hash: 19
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
- - 2
8
+ - 3
9
9
  - 0
10
- version: 0.2.0
10
+ version: 0.3.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Jeremy Cole
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-07-05 00:00:00 Z
18
+ date: 2012-07-16 00:00:00 Z
19
19
  dependencies: []
20
20
 
21
21
  description: Library for parsing MySQL binary logs in Ruby