mysql_binlog 0.1.0 → 0.1.1

data/lib/mysql_binlog/binlog_event_field_parser.rb

@@ -1,17 +1,21 @@
 module MysqlBinlog
+  # A mapping array for all values that may appear in the +status+ field of
+  # a query_event.
   QUERY_EVENT_STATUS_TYPES = [
-    :flags2,
-    :sql_mode,
-    :catalog,
-    :auto_increment,
-    :charset,
-    :time_zone,
-    :catalog_nz,
-    :lc_time_names,
-    :charset_database,
-    :table_map_for_update,
+    :flags2,                    # 0
+    :sql_mode,                  # 1
+    :catalog,                   # 2
+    :auto_increment,            # 3
+    :charset,                   # 4
+    :time_zone,                 # 5
+    :catalog_nz,                # 6
+    :lc_time_names,             # 7
+    :charset_database,          # 8
+    :table_map_for_update,      # 9
   ]
 
+  # A mapping hash for all values that may appear in the +flags2+ field of
+  # a query_event.
   QUERY_EVENT_FLAGS2 = {
     :auto_is_null           => 1 << 14,
     :not_autocommit         => 1 << 19,
@@ -31,26 +35,27 @@ module MysqlBinlog
       @table_map = {}
     end
 
+    # Parse additional fields for a +Rotate+ event.
     def rotate_event(header, fields)
       name_length = reader.remaining(header)
-      fields[:name_length] = name_length
       fields[:name] = reader.read(name_length)
     end
 
+    # Parse a dynamic +status+ structure within a query_event, which consists
+    # of a status_length (uint16) followed by a number of status variables
+    # (determined by the +status_length+) each of which consist of:
+    # * A type code (uint8), one of QUERY_EVENT_STATUS_TYPES.
+    # * The content itself, determined by the type. Additional processing is
+    #   required based on the type.
     def _query_event_status(header, fields)
       status = {}
-      end_position = reader.position + fields[:status_length]
+      status_length = parser.read_uint16
+      end_position = reader.position + status_length
       while reader.position < end_position
         status_type = QUERY_EVENT_STATUS_TYPES[parser.read_uint8]
         status[status_type] = case status_type
         when :flags2
-          flags2 = parser.read_uint32
-          QUERY_EVENT_FLAGS2.inject([]) do |result, (flag_name, flag_bit_value)|
-            if (flags2 & flag_bit_value) != 0
-              result << flag_name
-            end
-            result
-          end
+          parser.read_uint32_bitmap_by_name(QUERY_EVENT_FLAGS2)
         when :sql_mode
           parser.read_uint64
         when :catalog
@@ -81,15 +86,16 @@ module MysqlBinlog
       status
     end
 
+    # Parse additional fields for a +Query+ event.
     def query_event(header, fields)
       fields[:status] = _query_event_status(header, fields)
-      fields.delete :status_length
       fields[:db] = parser.read_nstringz(fields[:db_length])
       fields.delete :db_length
       query_length = reader.remaining(header)
       fields[:query] = reader.read([query_length, binlog.max_query_length].min)
     end
 
+    # Parse additional fields for an +Intvar+ event.
     def intvar_event(header, fields)
       case fields[:intvar_type]
       when 1
@@ -101,6 +107,7 @@ module MysqlBinlog
       end
     end
 
+    # Parse additional fields for a +Table_map+ event.
     def table_map_event(header, fields)
       fields[:table_id] = parser.read_uint48
       fields[:flags] = parser.read_uint16
@@ -122,6 +129,9 @@ module MysqlBinlog
       fields[:map_entry] = map_entry
     end
 
+    # Parse the row images present in a row-based replication row event. This
+    # is rather incomplete right now due missing support for many MySQL types,
+    # but can parse some basic events.
     def _generic_rows_event_row_images(header, fields)
       row_images = []
       end_position = reader.position + reader.remaining(header)
@@ -142,6 +152,10 @@ module MysqlBinlog
       row_images
     end
 
+    # Parse additional fields for any of the row-based replication row events:
+    # * +Write_rows+ which is used for +INSERT+.
+    # * +Update_rows+ which is used for +UPDATE+.
+    # * +Delete_rows+ which is used for +DELETE+.
     def generic_rows_event(header, fields)
       table_id = parser.read_uint48
       fields[:table] = @table_map[table_id]

data/lib/mysql_binlog/binlog_parser.rb

@@ -1,4 +1,5 @@
 module MysqlBinlog
+  # All MySQL types mapping to their integer values.
   MYSQL_TYPES_HASH = {
     :decimal         => 0,
     :tiny            => 1,
@@ -28,7 +29,8 @@ module MysqlBinlog
     :string          => 254,
     :geometry        => 255,
   }
-  
+
+  # All MySQL types in a simple lookup array to map an integer to its symbol.
   MYSQL_TYPES = MYSQL_TYPES_HASH.inject(Array.new(256)) do |type_array, item|
    type_array[item[1]] = item[0]
    type_array
@@ -44,76 +46,107 @@ module MysqlBinlog
       @reader = binlog_instance.reader
     end
 
+    # Read an unsigned 8-bit (1-byte) integer.
     def read_uint8
       reader.read(1).unpack("C").first
     end
 
+    # Read an unsigned 16-bit (2-byte) integer.
     def read_uint16
       reader.read(2).unpack("v").first
     end
 
+    # Read an unsigned 24-bit (3-byte) integer.
     def read_uint24
       a, b, c = reader.read(3).unpack("CCC")
       a + (b << 8) + (c << 16)
     end
 
+    # Read an unsigned 32-bit (4-byte) integer.
     def read_uint32
       reader.read(4).unpack("V").first
     end
 
+    # Read an unsigned 48-bit (6-byte) integer.
     def read_uint48
       a, b, c = reader.read(6).unpack("vvv")
       a + (b << 16) + (c << 32)
     end
 
+    # Read an unsigned 64-bit (8-byte) integer.
     def read_uint64
       reader.read(8).unpack("Q").first
     end
 
+    # Read a single-precision (4-byte) floating point number.
     def read_float
       reader.read(4).unpack("g").first
     end
 
+    # Read a double-precision (8-byte) floating point number.
     def read_double
       reader.read(8).unpack("G").first
     end
 
+    # Read a variable-length encoded integer. This is very broken at the
+    # moment, and is just mapping to read_uint8, so it cannot handle numbers
+    # greater than 251. This works fine for most structural elements of binary
+    # logs, but will fall over with decoding actual RBR row images.
     def read_varint
       # Cheating for now.
       read_uint8
     end
 
-    def read_lpstring
-      length = reader.read(1).unpack("C").first
+    # Read a non-terminated string, provided its length.
+    def read_nstring(length)
       reader.read(length)
     end
 
-    def read_lpstringz
-      string = read_lpstring
+    # Read a null-terminated string, provided its length (without the null).
+    def read_nstringz(length)
+      string = read_nstring(length)
       reader.read(1) # null
       string
     end
 
-    def read_nstring(length)
-      string = reader.read(length)
-      string
+    # Read a (Pascal-style) length-prefixed string. The length is stored as a
+    # 8-bit (1-byte) unsigned integer followed by the string itself with no
+    # termination character.
+    def read_lpstring
+      length = read_uint8
+      read_nstring(length)
     end
 
-    def read_nstringz(length)
-      string = reader.read(length)
-      reader.read(1) # null
-      string
+    # Read an lpstring which is also terminated with a null byte.
+    def read_lpstringz
+      length = read_uint8
+      read_nstringz(length)
     end
 
+    # Read an array of unsigned 8-bit (1-byte) integers.
     def read_uint8_array(length)
       reader.read(length).bytes.to_a
     end
 
+    # Read an arbitrary-length bitmap, provided its length. Returns an array
+    # of true/false values.
     def read_bit_array(length)
       data = reader.read((length+7)/8)
       data.unpack("b*").first.bytes.to_a.map { |i| (i-48) == 1 }.shift(length)
     end
 
+    def read_uint32_bitmap_by_name(names)
+      value = read_uint32
+      names.inject([]) do |result, (name, bit_value)|
+        if (value & bit_value) != 0
+          result << name
+        end
+        result
+      end
+    end
+
+    # Read a series of fields, provided an array of field descriptions. This
+    # can be used to read many types of fixed-length structures.
     def read_and_unpack(format_description)
       @format_cache[format_description] ||= {}
       this_format = @format_cache[format_description][:format] ||= 
@@ -131,6 +164,8 @@ module MysqlBinlog
       fields
     end
 
+    # Read a single field, provided the MySQL column type as a symbol. Not all
+    # types are currently supported.
     def read_mysql_type(column_type)
       case column_type
       #when :decimal

data/lib/mysql_binlog/mysql_binlog.rb

@@ -1,6 +1,7 @@
-require 'mysql_binlog/binlog_parser'
-
 module MysqlBinlog
+  # A simple method to print a string as in hex representation per byte,
+  # with no more than 24 bytes per line, and spaces between each byte.
+  # There is probably a better way to do this, but I don't know it.
   def puts_hex(data)
     hex = data.bytes.each_slice(24).inject("") do |string, slice|
       string << slice.map { |b| "%02x" % b }.join(" ") + "\n"
@@ -9,19 +10,7 @@ module MysqlBinlog
     puts hex
   end
 
-  MAGIC = [
-    { :name => :magic,            :length => 4,   :format => "V"   },
-  ]
-
-  EVENT_HEADER = [
-    { :name => :timestamp,        :length => 4,   :format => "V"   },
-    { :name => :event_type,       :length => 1,   :format => "C"   },
-    { :name => :server_id,        :length => 4,   :format => "V"   },
-    { :name => :event_length,     :length => 4,   :format => "V"   },
-    { :name => :next_position,    :length => 4,   :format => "V"   },
-    { :name => :flags,            :length => 2,   :format => "v"   },
-  ]
-
+  # An array to quickly map an integer event type to its symbol.
   EVENT_TYPES = [
     :unknown_event,             #  0
     :start_event_v3,            #  1
@@ -53,6 +42,19 @@ module MysqlBinlog
     :heartbeat_log_event,       # 27
   ]
 
+  # A common fixed-length header that is included with each event.
+  EVENT_HEADER = [
+    { :name => :timestamp,        :length => 4,   :format => "V"   },
+    { :name => :event_type,       :length => 1,   :format => "C"   },
+    { :name => :server_id,        :length => 4,   :format => "V"   },
+    { :name => :event_length,     :length => 4,   :format => "V"   },
+    { :name => :next_position,    :length => 4,   :format => "V"   },
+    { :name => :flags,            :length => 2,   :format => "v"   },
+  ]
+
+  # Values for the 'flags' field that may appear in binlogs. There are
+  # several other values that never appear in a file but may be used
+  # in events in memory.
   EVENT_FLAGS = {
     :binlog_in_use   => 0x01,
     :thread_specific => 0x04,
@@ -61,6 +63,9 @@ module MysqlBinlog
     :relay_log       => 0x40,
   }
 
+  # Format descriptions for fixed-length fields that may appear in the data
+  # for each event. Additional fields may be dynamic and are parsed by the
+  # methods in the BinlogEventFieldParser class.
   EVENT_FORMATS = {
     :format_description_event => [
       { :name => :binlog_version,   :length => 2,   :format => "v"   },
@@ -76,7 +81,6 @@ module MysqlBinlog
       { :name => :elapsed_time,     :length => 4,   :format => "V"   },
       { :name => :db_length,        :length => 1,   :format => "C"   },
       { :name => :error_code,       :length => 2,   :format => "v"   },
-      { :name => :status_length,    :length => 2,   :format => "v"   },
     ],
     :intvar_event => [
       { :name => :intvar_type,      :length => 1,   :format => "C"   },
@@ -115,49 +119,79 @@ module MysqlBinlog
       @max_query_length = 1048576
     end
 
+    # Rewind to the beginning of the log, if supported by the reader. The
+    # reader may throw an exception if rewinding is not supported (e.g. for
+    # a stream-based reader).
     def rewind
       reader.rewind
-      read_file_header
     end
 
+    # Read fixed fields using format definitions in 'unpack' format provided
+    # in the EVENT_FORMATS hash.
+    def read_fixed_fields(event_type, header)
+      if EVENT_FORMATS.include? event_type
+        parser.read_and_unpack(EVENT_FORMATS[event_type])
+      end
+    end
+
+    # Read dynamic fields or fields that require more processing before being
+    # saved in the event.
     def read_additional_fields(event_type, header, fields)
       if event_field_parser.methods.include? event_type.to_s
         event_field_parser.send(event_type, header, fields)
       end
     end
 
+    # Skip the remainder of this event. This can be used to skip an entire
+    # event or merely the parts of the event this library does not understand.
     def skip_event(header)
       reader.skip(header)
     end
 
+    # Read the common header for an event. Every event has a header.
     def read_event_header
       header = parser.read_and_unpack(EVENT_HEADER)
+
+      # Merge the read 'flags' bitmap with the EVENT_FLAGS hash to return
+      # the flags by name instead of returning the bitmap as an integer.
       flags = EVENT_FLAGS.inject([]) do |result, (flag_name, flag_bit_value)|
         if (header[:flags] & flag_bit_value) != 0
           result << flag_name
         end
         result
       end
+
+      # Overwrite the integer version of 'flags' with the array of names.
       header[:flags] = flags
+
       header
     end
 
+    # Read the content of the event, which consists of an optional fixed field
+    # portion and an optional dynamic portion.
     def read_event_content(header)
-      content = nil
-
       event_type = EVENT_TYPES[header[:event_type]]
-      if EVENT_FORMATS.include? event_type
-        content = parser.read_and_unpack(EVENT_FORMATS[event_type])
-      else
-        content = {}
-      end
 
+      # Read the fixed portion of the event, if it is understood, or there is
+      # one. If not, initialize content with an empty hash instead.
+      content = read_fixed_fields(event_type, header) || {}
+
+      # Read additional fields from the dynamic portion of the event. Some of
+      # these may actually be fixed width but needed more processing in a
+      # function instead of the unpack formats possible in read_fixed_fields.
       read_additional_fields(event_type, header, content)
 
+      # Anything left unread at this point is skipped based on the event length
+      # provided in the header. In this way, it is possible to skip over events
+      # that are not able to be parsed correctly by this library.
       skip_event(header)
+
       content
     end
 
+    # Scan events until finding one that isn't rejected by the filter rules.
+    # If there are no filter rules, this will return the next event provided
+    # by the reader.
     def read_event
       while true
         skip_this_event = false
@@ -187,12 +221,13 @@ module MysqlBinlog
           end
         end
 
+        # Never skip over rotate_event or format_description_event as they
+        # are critical to understanding the format of this event stream.
         unless [:rotate_event, :format_description_event].include? event_type
           next if skip_this_event
         end
         
         content = read_event_content(header)
-        content
 
         case event_type
         when :rotate_event
@@ -213,6 +248,9 @@ module MysqlBinlog
       }
     end
 
+    # Process a format description event, which describes the version of this
+    # file, and the format of events which will appear in this file. This also
+    # provides the version of the MySQL server which generated this file.
     def process_fde(fde)
       if (version = fde[:binlog_version]) != 4
         raise UnsupportedVersionException.new("Binlog version #{version} is not supported")
@@ -224,7 +262,8 @@ module MysqlBinlog
         :server_version => fde[:server_version],
       }
     end
-  
+
+    # Iterate through all events.
     def each_event
       while event = read_event
         yield event

data/lib/mysql_binlog.rb

@@ -1,4 +1,5 @@
 require 'mysql_binlog/mysql_binlog'
+require 'mysql_binlog/binlog_parser'
 require 'mysql_binlog/binlog_event_field_parser'
 require 'mysql_binlog/reader/binlog_file_reader'
 require 'mysql_binlog/reader/binlog_stream_reader'

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mysql_binlog
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: 
   segments: 
   - 0
   - 1
-  - 0
-  version: 0.1.0
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - Jeremy Cole