mysql_binlog 0.2.0 → 0.3.0

data/bin/mysql_binlog_dump

@@ -1,16 +1,100 @@
 #!/usr/bin/env ruby
 
-require 'pp'
 require 'mysql_binlog'
+require 'bigdecimal'
+require 'getoptlong'
+require 'ostruct'
+require 'pp'
 
 include MysqlBinlog
 
-reader = BinlogFileReader.new(ARGV.first)
-#reader = DebuggingReader.new(reader, :data => true, :calls => true)
+def usage(exit_code, message = nil)
+  print "Error: #{message}\n\n" unless message.nil?
+
+  print <<'END_OF_USAGE'
+
+Usage:
+  To read from a binary log file on disk:
+    mysql_binlog_dump [options] -f <filename>
+
+  --help, -?
+    Show this help.
+
+  --file, -f <filename>
+    Read from a binary log file on disk.
+
+  --debug, -d
+    Debug reading from the binary log, showing calls into the reader and the
+    data bytes read. This is useful for debugging the mysql_binlog library
+    as well as debugging problems with binary logs.
+
+  --tail, -t
+    When reading from a file, follow the end of the binary log file instead
+    of exiting when reaching the end. Exit with Control-C.
+
+  --rotate, -r
+    When reading from a file, follow the rotate events which may be at the
+    end of a file (due to log rotation) so that the stream can be followed
+    through multiple files. This is especially useful with --tail.
+
+END_OF_USAGE
+
+  exit exit_code
+end
+
+
+@options = OpenStruct.new
+@options.file = nil
+@options.debug = false
+@options.tail = false
+@options.rotate = false
+
+getopt_options = [
+  [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
+  [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--debug",            "-d",     GetoptLong::NO_ARGUMENT ],
+  [ "--tail",             "-t",     GetoptLong::NO_ARGUMENT ],
+  [ "--rotate",           "-r",     GetoptLong::NO_ARGUMENT ],
+]
+
+getopt = GetoptLong.new(*getopt_options)
+
+getopt.each do |opt, arg|
+  case opt
+  when "--help"
+    usage 0
+  when "--file"
+    @options.file = arg
+  when "--debug"
+    @options.debug = true
+  when "--tail"
+    @options.tail = true
+  when "--rotate"
+    @options.rotate = true
+  end
+end
+
+unless @options.file
+  usage 1, "A file must be provided with --file/-f"
+end
+
+reader = BinlogFileReader.new(@options.file)
+if @options.debug
+  reader = DebuggingReader.new(reader, :data => true, :calls => true)
+end
 binlog = Binlog.new(reader)
 
-#reader.tail = true
-#binlog.ignore_rotate = true
+if @options.tail
+  reader.tail = true
+else
+  reader.tail = false
+end
+
+if @options.rotate
+  binlog.ignore_rotate = false
+else
+  binlog.ignore_rotate = true
+end
 
 binlog.each_event do |event|
   pp event

data/lib/mysql_binlog/binlog.rb

@@ -2,6 +2,9 @@ module MysqlBinlog
   # This version of the binary log format is not supported by this library.
   class UnsupportedVersionException < Exception; end
 
+  # This field type is not supported by this library.
+  class UnsupportedTypeException < Exception; end
+
   # An error was encountered when trying to read the log, which was likely
   # due to garbage data in the log. Continuing is likely impossible.
   class MalformedBinlogException < Exception; end
@@ -141,8 +144,9 @@ module MysqlBinlog
 
         case header[:event_type]
         when :rotate_event
-          next if ignore_rotate
-          reader.rotate(fields[:name], fields[:pos])
+          unless ignore_rotate
+            reader.rotate(fields[:name], fields[:pos])
+          end
         when :format_description_event
           process_fde(fields)
         end

data/lib/mysql_binlog/binlog_event_parser.rb

@@ -1,36 +1,52 @@
 module MysqlBinlog
-  # An array to quickly map an integer event type to its symbol.
+  # A hash of all possible event type IDs.
   #
   # Enumerated in sql/log_event.h line ~539 as Log_event_type
-  EVENT_TYPES = [
-    :unknown_event,             #  0
-    :start_event_v3,            #  1 (deprecated)
-    :query_event,               #  2
-    :stop_event,                #  3
-    :rotate_event,              #  4
-    :intvar_event,              #  5
-    :load_event,                #  6 (deprecated)
-    :slave_event,               #  7 (deprecated)
-    :create_file_event,         #  8 (deprecated)
-    :append_block_event,        #  9
-    :exec_load_event,           # 10 (deprecated)
-    :delete_file_event,         # 11
-    :new_load_event,            # 12 (deprecated)
-    :rand_event,                # 13
-    :user_var_event,            # 14
-    :format_description_event,  # 15
-    :xid_event,                 # 16
-    :begin_load_query_event,    # 17
-    :execute_load_query_event,  # 18
-    :table_map_event,           # 19
-    :pre_ga_write_rows_event,   # 20 (deprecated)
-    :pre_ga_update_rows_event,  # 21 (deprecated)
-    :pre_ga_delete_rows_event,  # 22 (deprecated)
-    :write_rows_event,          # 23
-    :update_rows_event,         # 24
-    :delete_rows_event,         # 25
-    :incident_event,            # 26
-    :heartbeat_log_event,       # 27
+  EVENT_TYPES_HASH = {
+    :unknown_event              =>  0,  #
+    :start_event_v3             =>  1,  # (deprecated)
+    :query_event                =>  2,  #
+    :stop_event                 =>  3,  #
+    :rotate_event               =>  4,  #
+    :intvar_event               =>  5,  #
+    :load_event                 =>  6,  # (deprecated)
+    :slave_event                =>  7,  # (deprecated)
+    :create_file_event          =>  8,  # (deprecated)
+    :append_block_event         =>  9,  #
+    :exec_load_event            => 10,  # (deprecated)
+    :delete_file_event          => 11,  #
+    :new_load_event             => 12,  # (deprecated)
+    :rand_event                 => 13,  #
+    :user_var_event             => 14,  #
+    :format_description_event   => 15,  #
+    :xid_event                  => 16,  #
+    :begin_load_query_event     => 17,  #
+    :execute_load_query_event   => 18,  #
+    :table_map_event            => 19,  #
+    :pre_ga_write_rows_event    => 20,  # (deprecated)
+    :pre_ga_update_rows_event   => 21,  # (deprecated)
+    :pre_ga_delete_rows_event   => 22,  # (deprecated)
+    :write_rows_event           => 23,  #
+    :update_rows_event          => 24,  #
+    :delete_rows_event          => 25,  #
+    :incident_event             => 26,  #
+    :heartbeat_log_event        => 27,  #
+    :table_metadata_event       => 50,  # Only in Twitter MySQL
+  }
+
+  # A lookup array to map an integer event type ID to its symbol.
+  EVENT_TYPES = EVENT_TYPES_HASH.inject(Array.new(256)) do |type_array, item|
+   type_array[item[1]] = item[0]
+   type_array
+  end
+
+  # A list of supported row-based replication event types. Since these all
+  # have an identical structure, this list can be used by other programs to
+  # know which events can be treated as row events.
+  ROW_EVENT_TYPES = [
+    :write_rows_event,
+    :update_rows_event,
+    :delete_rows_event,
   ]
 
   # Values for the +flags+ field that may appear in binary logs. There are
@@ -96,6 +112,35 @@ module MysqlBinlog
     :bit_len_exact          => 1 << 0,  # TM_BIT_LEN_EXACT_F
   }
 
+  # A mapping array for all values that may appear in the +flags+ field of a
+  # table_metadata_event.
+  #
+  # There are none of these at the moment.
+  TABLE_METADATA_EVENT_FLAGS = {
+  }
+
+  # A mapping hash for all values that may appear in the +flags+ field of
+  # a column descriptor for a table_metadata_event.
+  #
+  # Defined in include/mysql_com.h line ~92
+  TABLE_METADATA_EVENT_COLUMN_FLAGS = {
+    :not_null             =>     1, # NOT_NULL_FLAG
+    :primary_key          =>     2, # PRI_KEY_FLAG
+    :unique_key           =>     4, # UNIQUE_KEY_FLAG
+    :multiple_key         =>     8, # MULTIPLE_KEY_FLAG
+    :blob                 =>    16, # BLOB_FLAG
+    :unsigned             =>    32, # UNSIGNED_FLAG
+    :zerofill             =>    64, # ZEROFILL_FLAG
+    :binary               =>   128, # BINARY_FLAG
+    :enum                 =>   256, # ENUM_FLAG
+    :auto_increment       =>   512, # AUTO_INCREMENT_FLAG
+    :timestamp            =>  1024, # TIMESTAMP_FLAG
+    :set                  =>  2048, # SET_FLAG
+    :no_default_value     =>  4096, # NO_DEFAULT_VALUE_FLAG
+    :on_update_now        =>  8192, # ON_UPDATE_NOW_FLAG
+    :part_key             => 16384, # PART_KEY_FLAG
+  }
+
   # A mapping array for all values that may appear in the +flags+ field of a
   # write_rows_event, update_rows_event, or delete_rows_event.
   #
@@ -296,12 +341,14 @@ module MysqlBinlog
         # which is fundamentally incompatible with :string parsing. Setting
         # a :type key in this hash will cause table_map_event to override the
         # main field :type with the provided type here.
-        real_type = MYSQL_TYPES[parser.read_uint8]
+        # See Field_string::do_save_field_metadata for reference.
+        metadata  = (parser.read_uint8 << 8) + parser.read_uint8
+        real_type = MYSQL_TYPES[metadata >> 8]
         case real_type
         when :enum, :set
-          { :type => real_type, :size => parser.read_uint8 }
+          { :type => real_type, :size => metadata & 0x00ff }
         else
-          { :max_length  => parser.read_uint8 }
+          { :max_length  => (((metadata >> 4) & 0x300) ^ 0x300) + (metadata & 0x00ff) }
         end
       end
     end
@@ -353,6 +400,36 @@ module MysqlBinlog
       fields
     end
 
+    # Parse fields for a +Table_metadata+ event, which is specific to
+    # Twitter MySQL releases at the moment.
+    #
+    # Implemented in sql/log_event.cc line ~8772 (in Twitter MySQL)
+    # in Table_metadata_log_event::write_data_header
+    # and Table_metadata_log_event::write_data_body
+    def table_metadata_event(header)
+      fields = {}
+      table_id = parser.read_uint48
+      columns = parser.read_uint16
+
+      fields[:table] = @table_map[table_id]
+      fields[:flags] = parser.read_uint16
+      fields[:columns] = columns.times.map do |c|
+        descriptor_length = parser.read_uint32
+        @table_map[table_id][:columns][c][:description] = {
+          :type => MYSQL_TYPES[parser.read_uint8],
+          :length => parser.read_uint32,
+          :scale => parser.read_uint8,
+          :character_set => COLLATION[parser.read_uint16],
+          :flags => parser.read_uint_bitmap_by_size_and_name(2,
+                    TABLE_METADATA_EVENT_COLUMN_FLAGS),
+          :name => parser.read_varstring,
+          :type_name => parser.read_varstring,
+          :comment => parser.read_varstring,
+        }
+      end
+      fields
+    end
+
     # Parse a single row image, which is comprised of a series of columns. Not
     # all columns are present in the row image, the columns_used array of true
     # and false values identifies which columns are present.
@@ -363,10 +440,11 @@ module MysqlBinlog
         if !columns_used[column_index]
           row_image << nil
         elsif columns_null[column_index]
-          row_image << { column => nil }
+          row_image << { column_index => nil }
         else
           row_image << {
-            column => parser.read_mysql_type(column[:type], column[:metadata])
+            column_index =>
+              parser.read_mysql_type(column[:type], column[:metadata])
           }
         end
       end
@@ -438,4 +516,4 @@ module MysqlBinlog
     alias :delete_rows_event :generic_rows_event
 
   end
-end
+end

data/lib/mysql_binlog/binlog_field_parser.rb

@@ -69,17 +69,54 @@ module MysqlBinlog
       reader.read(4).unpack("V").first
     end
 
+    # Read an unsigned 40-bit (5-byte) integer.
+    def read_uint40
+      a, b = reader.read(5).unpack("CV")
+      a + (b << 8)
+    end
+
     # Read an unsigned 48-bit (6-byte) integer.
     def read_uint48
       a, b, c = reader.read(6).unpack("vvv")
       a + (b << 16) + (c << 32)
     end
 
+    # Read an unsigned 56-bit (7-byte) integer.
+    def read_uint56
+      a, b, c = reader.read(7).unpack("CvV")
+      a + (b << 8) + (c << 24)
+    end
+
     # Read an unsigned 64-bit (8-byte) integer.
     def read_uint64
       reader.read(8).unpack("Q").first
     end
 
+    # Read a signed 8-bit (1-byte) integer.
+    def read_int8
+      reader.read(1).unpack("c").first
+    end
+
+    # Read a signed 16-bit (2-byte) big-endian integer.
+    def read_int16_be
+      reader.read(2).reverse.unpack('s').first
+    end
+
+    # Read a signed 24-bit (3-byte) big-endian integer.
+    def read_int24_be
+      a, b, c = reader.read(3).unpack('CCC')
+      if a & 128
+        (a << 16) | (b << 8) | c
+      else
+        (-1 << 24) | (a << 16) | (b << 8) | c
+      end
+    end
+
+    # Read a signed 32-bit (4-byte) big-endian integer.
+    def read_int32_be
+      reader.read(4).reverse.unpack('l').first
+    end
+
     def read_uint_by_size(size)
       case size
       when 1
@@ -90,13 +127,32 @@ module MysqlBinlog
         read_uint24
       when 4
         read_uint32
+      when 5
+        read_uint40
       when 6
         read_uint48
+      when 7
+        read_uint56
       when 8
         read_uint64
       end
     end
 
+    def read_int_be_by_size(size)
+      case size
+      when 1
+        read_int8
+      when 2
+        read_int16_be
+      when 3
+        read_int24_be
+      when 4
+        read_int32_be
+      else
+        raise "read_int#{size*8}_be not implemented"
+      end
+    end
+
     # Read a single-precision (4-byte) floating point number.
     def read_float
       reader.read(4).unpack("e").first
@@ -168,6 +224,57 @@ module MysqlBinlog
       read_nstring(length)
     end
 
+    # Read a (new) decimal value. The value is stored as a sequence of signed
+    # big-endian integers, each representing up to 9 digits of the integral
+    # and fractional parts. The first integer of the integral part and/or the
+    # last integer of the fractional part might be compressed (or packed) and
+    # are of variable length. The remaining integers (if any) are
+    # uncompressed and 32 bits wide.
+    def read_newdecimal(precision, scale)
+      digits_per_integer = 9
+      compressed_bytes = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4]
+      integral = (precision - scale)
+      uncomp_integral = integral / digits_per_integer
+      uncomp_fractional = scale / digits_per_integer
+      comp_integral = integral - (uncomp_integral * digits_per_integer)
+      comp_fractional = scale - (uncomp_fractional * digits_per_integer)
+
+      # The sign is encoded in the high bit of the first byte/digit. The byte
+      # might be part of a larger integer, so apply the optional bit-flipper
+      # and push back the byte into the input stream.
+      value = read_uint8
+      str, mask = (value & 0x80 != 0) ? ["", 0] : ["-", -1]
+      reader.unget(value ^ 0x80)
+
+      size = compressed_bytes[comp_integral]
+
+      if size > 0
+        value = read_int_be_by_size(size) ^ mask
+        str << value.to_s
+      end
+
+      (1..uncomp_integral).each do
+        value = read_int32_be ^ mask
+        str << value.to_s
+      end
+
+      str << "."
+
+      (1..uncomp_fractional).each do
+        value = read_int32_be ^ mask
+        str << value.to_s
+      end
+
+      size = compressed_bytes[comp_fractional]
+
+      if size > 0
+        value = read_int_be_by_size(size) ^ mask
+        str << value.to_s
+      end
+
+      BigDecimal.new(str)
+    end
+
     # Read an array of unsigned 8-bit (1-byte) integers.
     def read_uint8_array(length)
       reader.read(length).bytes.to_a
@@ -178,20 +285,36 @@ module MysqlBinlog
     # events that need bitmaps, as well as for the BIT type.
     def read_bit_array(length)
       data = reader.read((length+7)/8)
-      data.unpack("B*").first.reverse.  # Unpack into a string of "10101"
+      data.unpack("b*").first.  # Unpack into a string of "10101"
         split("").map { |c| c == "1" }.shift(length) # Return true/false array
     end
 
     # Read a uint value using the provided size, and convert it to an array
     # of symbols derived from a mapping table provided.
-    def read_uint_bitmap_by_size_and_name(size, names)
+    def read_uint_bitmap_by_size_and_name(size, bit_names)
       value = read_uint_by_size(size)
-      names.inject([]) do |result, (name, bit_value)|
+      named_bits = []
+
+      # Do an efficient scan for the named bits we know about using the hash
+      # provided.
+      bit_names.each do |(name, bit_value)|
         if (value & bit_value) != 0
-          result << name
+          value -= bit_value
+          named_bits << name
         end
-        result
       end
+
+      # If anything is left over in +value+, add "unknown" names to the result
+      # so that they can be identified and corrected.
+      if value > 0
+        0.upto(size * 8).map { |n| 1 << n }.each do |bit_value|
+          if (value & bit_value) != 0
+            named_bits << "unknown_#{bit_value}".to_sym
+          end
+        end
+      end
+
+      named_bits
     end
 
     # Extract a number of sequential bits at a given offset within an integer.
@@ -254,10 +377,11 @@ module MysqlBinlog
         read_float
       when :double
         read_double
-      when :string, :var_string
+      when :var_string
         read_varstring
-      when :varchar
-        read_lpstring(2)
+      when :varchar, :string
+        prefix_size = (metadata[:max_length] > 255) ? 2 : 1
+        read_lpstring(prefix_size)
       when :blob, :geometry
         read_lpstring(metadata[:length_size])
       when :timestamp
@@ -273,9 +397,15 @@ module MysqlBinlog
       when :enum, :set
         read_uint_by_size(metadata[:size])
       when :bit
-        read_bit_array(metadata[:bits])
-      #when :newdecimal
+        byte_length = (metadata[:bits]+7)/8
+        read_uint_by_size(byte_length)
+      when :newdecimal
+        precision = metadata[:precision]
+        scale = metadata[:decimals]
+        read_newdecimal(precision, scale)
+      else
+        raise UnsupportedTypeException.new("Type #{type} is not supported.")
       end
     end
   end
-end
+end

data/lib/mysql_binlog/reader/binlog_file_reader.rb

@@ -58,7 +58,11 @@ module MysqlBinlog
     def seek(pos)
       @binlog.seek(pos)
     end
-  
+
+    def unget(char)
+      @binlog.ungetc(char)
+    end
+
     def end?
       return false if tail
       @binlog.eof?
@@ -91,4 +95,4 @@ module MysqlBinlog
       data
     end
   end
-end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mysql_binlog
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 19
   prerelease: 
   segments: 
   - 0
-  - 2
+  - 3
   - 0
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Jeremy Cole
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-05 00:00:00 Z
+date: 2012-07-16 00:00:00 Z
 dependencies: []
 
 description: Library for parsing MySQL binary logs in Ruby