mysql_binlog 0.1.1 → 0.1.2

data/lib/mysql_binlog/binlog_event_field_parser.rb

@@ -4,11 +4,11 @@ module MysqlBinlog
   QUERY_EVENT_STATUS_TYPES = [
     :flags2,                    # 0
     :sql_mode,                  # 1
-    :catalog,                   # 2
+    :catalog_deprecated,        # 2
     :auto_increment,            # 3
     :charset,                   # 4
     :time_zone,                 # 5
-    :catalog_nz,                # 6
+    :catalog,                   # 6
     :lc_time_names,             # 7
     :charset_database,          # 8
     :table_map_for_update,      # 9
@@ -58,7 +58,7 @@ module MysqlBinlog
           parser.read_uint32_bitmap_by_name(QUERY_EVENT_FLAGS2)
         when :sql_mode
           parser.read_uint64
-        when :catalog
+        when :catalog_deprecated
           parser.read_lpstringz
         when :auto_increment
           {
@@ -73,7 +73,7 @@ module MysqlBinlog
           }
         when :time_zone
           parser.read_lpstring
-        when :catalog_nz
+        when :catalog
           parser.read_lpstring
         when :lc_time_names
           parser.read_uint16
@@ -107,6 +107,13 @@ module MysqlBinlog
       end
     end
 
+    def _table_map_event_metadata(columns_type)
+      length = parser.read_varint
+      columns_type.map do |c|
+        parser.read_mysql_type_metadata(c)
+      end
+    end
+
     # Parse additional fields for a +Table_map+ event.
     def table_map_event(header, fields)
       fields[:table_id] = parser.read_uint48
@@ -116,13 +123,14 @@ module MysqlBinlog
       map_entry[:table] = parser.read_lpstringz
       columns = parser.read_varint
       columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] }
-      columns_metadata = parser.read_lpstring
+      columns_metadata = _table_map_event_metadata(columns_type)
       columns_nullable = parser.read_bit_array(columns)
 
       map_entry[:columns] = columns.times.map do |c|
         {
           :type     => columns_type[c],
           :nullable => columns_nullable[c],
+          :metadata => columns_metadata[c],
         }
       end
 
@@ -132,22 +140,26 @@ module MysqlBinlog
     # Parse the row images present in a row-based replication row event. This
     # is rather incomplete right now due missing support for many MySQL types,
     # but can parse some basic events.
-    def _generic_rows_event_row_images(header, fields)
+    def _generic_rows_event_row_images(header, fields, columns_used)
+      row_image_index = 0
       row_images = []
       end_position = reader.position + reader.remaining(header)
       while reader.position < end_position
         row_image = []
         columns_null = parser.read_bit_array(fields[:table][:columns].size)
         fields[:table][:columns].each_with_index do |column, column_index|
-          if !fields[:columns_used][column_index]
+          if !columns_used[row_image_index][column_index]
             row_image << nil
           elsif columns_null[column_index]
             row_image << { column => nil }
           else
-            row_image << { column => parser.read_mysql_type(column[:type]) }
+            row_image << {
+              column => parser.read_mysql_type(column[:type], column[:metadata])
+            }
           end
         end
         row_images << row_image
+        row_image_index += 1
       end
       row_images
     end
@@ -161,12 +173,12 @@ module MysqlBinlog
       fields[:table] = @table_map[table_id]
       fields[:flags] = parser.read_uint16
       columns = parser.read_varint
-      fields[:columns_used] = parser.read_bit_array(columns)
+      columns_used = []
+      columns_used[0] = parser.read_bit_array(columns)
       if EVENT_TYPES[header[:event_type]] == :update_rows_event
-        fields[:columns_update] = parser.read_bit_array(columns)
+        columns_used[1] = parser.read_bit_array(columns)
       end
-      fields[:row_image] = _generic_rows_event_row_images(header, fields)
-      fields.delete :columns_used
+      fields[:row_image] = _generic_rows_event_row_images(header, fields, columns_used)
     end
     alias :write_rows_event  :generic_rows_event
     alias :update_rows_event :generic_rows_event

data/lib/mysql_binlog/binlog_parser.rb

@@ -88,13 +88,31 @@ module MysqlBinlog
       reader.read(8).unpack("G").first
     end
 
-    # Read a variable-length encoded integer. This is very broken at the
-    # moment, and is just mapping to read_uint8, so it cannot handle numbers
-    # greater than 251. This works fine for most structural elements of binary
-    # logs, but will fall over with decoding actual RBR row images.
+    # Read a variable-length "Length Coded Binary" integer. This is derived
+    # from the MySQL protocol, and re-used in the binary log format. This
+    # format uses the first byte to alternately store the actual value for
+    # integer values <= 250, or to encode the number of following bytes
+    # used to store the actual value, which can be 2, 3, or 8. It also
+    # includes support for SQL NULL as a special case.
+    #
+    # See: http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Elements
     def read_varint
-      # Cheating for now.
-      read_uint8
+      first_byte = read_uint8
+
+      case
+      when first_byte <= 250
+        first_byte
+      when first_byte == 251
+        nil
+      when first_byte == 252
+        read_uint16
+      when first_byte == 253
+        read_uint24
+      when first_byte == 254
+        read_uint64
+      when first_byte == 255
+        raise "Invalid variable-length integer"
+      end
     end
 
     # Read a non-terminated string, provided its length.
@@ -110,17 +128,36 @@ module MysqlBinlog
     end
 
     # Read a (Pascal-style) length-prefixed string. The length is stored as a
-    # 8-bit (1-byte) unsigned integer followed by the string itself with no
-    # termination character.
-    def read_lpstring
-      length = read_uint8
+    # 8-bit (1-byte) to 32-bit (4-byte) unsigned integer, depending on the
+    # optional size parameter (default 1), followed by the string itself with
+    # no termination character.
+    def read_lpstring(size=1)
+      case size
+      when 1
+        length = read_uint8
+      when 2
+        length = read_uint16
+      when 3
+        length = read_uint24
+      when 4
+        length = read_uint32
+      end
       read_nstring(length)
     end
 
-    # Read an lpstring which is also terminated with a null byte.
-    def read_lpstringz
-      length = read_uint8
-      read_nstringz(length)
+    # Read an lpstring (as above) which is also terminated with a null byte.
+    def read_lpstringz(size=1)
+      string = read_lpstring(size)
+      reader.read(1) # null
+      string
+    end
+
+    # Read a MySQL-style varint length-prefixed string. The length is stored
+    # as a variable-length "Length Coded Binary" value (see read_varint) which
+    # is followed by the string content itself. No termination is included.
+    def read_varstring
+      length = read_varint
+      read_nstring(length)
     end
 
     # Read an array of unsigned 8-bit (1-byte) integers.
@@ -135,6 +172,8 @@ module MysqlBinlog
       data.unpack("b*").first.bytes.to_a.map { |i| (i-48) == 1 }.shift(length)
     end
 
+    # Read a uint32 value, and convert it to an array of symbols derived
+    # from a mapping table provided.
     def read_uint32_bitmap_by_name(names)
       value = read_uint32
       names.inject([]) do |result, (name, bit_value)|
@@ -164,11 +203,35 @@ module MysqlBinlog
       fields
     end
 
+    def read_mysql_type_metadata(column_type)
+      case column_type
+      when :float, :double
+        { :size => read_uint8 }
+      when :varchar
+        { :max_length => read_uint16 }
+      when :bit
+        { :size => read_uint8 }
+      when :decimal
+        {
+          :precision => read_uint8,
+          :decimals  => read_uint8,
+        }
+      when :blob
+        { :length_size => read_uint8 }
+      when :var_string, :string
+        {
+          :real_type   => read_uint8,
+          :length_size => read_uint8,
+        }
+      when :geometry
+        { :length_size => read_uint8 }
+      end
+    end
+
     # Read a single field, provided the MySQL column type as a symbol. Not all
     # types are currently supported.
-    def read_mysql_type(column_type)
-      case column_type
-      #when :decimal
+    def read_mysql_type(type, metadata=nil)
+      case type
       when :tiny
         read_uint8
       when :short
@@ -179,33 +242,27 @@ module MysqlBinlog
         read_uint32
       when :longlong
         read_uint64
-      when :string
-        length = read_varint
-        read_nstring(length)
-
       when :float
         read_float
       when :double
         read_double
-      #when :null
+      when :string, :var_string
+        read_varstring
+      when :varchar, :blob
+        read_lpstring(metadata[:length_size])
       when :timestamp
         read_uint32
+      when :year
+        read_uint8 + 1900
       #when :date
       #when :time
       #when :datetime
-      #when :year
       #when :newdate
-      #when :varchar
       #when :bit
+      #when :decimal
       #when :newdecimal
       #when :enum
       #when :set
-      #when :tiny_blob
-      #when :medium_blob
-      #when :long_blob
-      #when :blob
-      #when :var_string
-      #when :string
       #when :geometry
       end
     end

data/lib/mysql_binlog/reader/binlog_file_reader.rb

@@ -52,6 +52,7 @@ module MysqlBinlog
     def read(length)
       return "" if length == 0
       data = @binlog.read(length)
+      puts_hex data
       if !data
         raise MalformedBinlogException.new
       elsif data.length == 0

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mysql_binlog
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 31
   prerelease: 
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Jeremy Cole