mysql_binlog 0.1.8 → 0.2.0

data/lib/mysql_binlog/binlog.rb

@@ -14,6 +14,13 @@ module MysqlBinlog
   # event.
   class ShortReadException < Exception; end
 
+  # After an event or other structure was fully read, the log position exceeded
+  # the end of the structure being read. This would indicate a bug in parsing
+  # the fields in the structure. For example, reading garbage data for a length
+  # field may cause a string read based on that length to read data well past
+  # the end of the event or structure. This is essentially always fatal.
+  class OverReadException < Exception; end
+
   # Read a binary log, parsing and returning events.
   #
   # == Examples
@@ -69,17 +76,24 @@ module MysqlBinlog
 
     # Read the content of the event, which follows the header.
     def read_event_fields(header)
-      event_type = EVENT_TYPES[header[:event_type]]
-
       # Delegate the parsing of the event content to a method of the same name
       # in BinlogEventParser.
-      if event_parser.methods.include? event_type.to_s
-        fields = event_parser.send(event_type, header)
+      if event_parser.methods.include? header[:event_type].to_s
+        fields = event_parser.send(header[:event_type], header)
+      end
+
+      # Check if we've read past the end of the event. This is normally because
+      # of an unsupported substructure in the event causing field misalignment
+      # or a bug in the event reader method in BinlogEventParser. This may also
+      # be due to user error in providing an initial start position or later
+      # seeking to a position which is not a valid event start position.
+      if reader.position > header[:next_position]
+        raise OverReadException.new("Read past end of event; corrupted event, bad start position, or bug in mysql_binlog?")
       end
 
       # Anything left unread at this point is skipped based on the event length
       # provided in the header. In this way, it is possible to skip over events
-      # that are not able to be parsed correctly by this library.
+      # that are not able to be parsed completely by this library.
       skip_event(header)
 
       fields
@@ -102,10 +116,8 @@ module MysqlBinlog
           return nil
         end
 
-        event_type = EVENT_TYPES[header[:event_type]]
-        
         if @filter_event_types
-          unless @filter_event_types.include? event_type
+          unless @filter_event_types.include? header[:event_type]
             skip_this_event = true
           end
         end
@@ -119,15 +131,15 @@ module MysqlBinlog
         # Never skip over rotate_event or format_description_event as they
         # are critical to understanding the format of this event stream.
         if skip_this_event
-          unless [:rotate_event, :format_description_event].include? event_type
+          unless [:rotate_event, :format_description_event].include? header[:event_type]
             skip_event(header)
             next
           end
         end
-        
+
         fields = read_event_fields(header)
 
-        case event_type
+        case header[:event_type]
         when :rotate_event
           next if ignore_rotate
           reader.rotate(fields[:name], fields[:pos])
@@ -139,7 +151,7 @@ module MysqlBinlog
       end
 
       {
-        :type     => event_type,
+        :type     => header[:event_type],
         :filename => filename,
         :position => position,
         :header   => header,
@@ -167,6 +179,10 @@ module MysqlBinlog
 
     # Iterate through all events.
     def each_event
+      unless block_given?
+        return Enumerable::Enumerator.new(self, :each_event)
+      end
+
       while event = read_event
         yield event
       end

data/lib/mysql_binlog/binlog_event_parser.rb

@@ -1,40 +1,21 @@
 module MysqlBinlog
-  # A common fixed-length header that is included with each event.
-  EVENT_HEADER = [
-    { :name => :timestamp,        :length => 4,   :format => "V"   },
-    { :name => :event_type,       :length => 1,   :format => "C"   },
-    { :name => :server_id,        :length => 4,   :format => "V"   },
-    { :name => :event_length,     :length => 4,   :format => "V"   },
-    { :name => :next_position,    :length => 4,   :format => "V"   },
-    { :name => :flags,            :length => 2,   :format => "v"   },
-  ]
-
-  # Values for the +flags+ field that may appear in binary logs. There are
-  # several other values that never appear in a file but may be used
-  # in events in memory.
-  EVENT_HEADER_FLAGS = {
-    :binlog_in_use   => 0x01,
-    :thread_specific => 0x04,
-    :suppress_use    => 0x08,
-    :artificial      => 0x20,
-    :relay_log       => 0x40,
-  }
-
   # An array to quickly map an integer event type to its symbol.
+  #
+  # Enumerated in sql/log_event.h line ~539 as Log_event_type
   EVENT_TYPES = [
     :unknown_event,             #  0
-    :start_event_v3,            #  1
+    :start_event_v3,            #  1 (deprecated)
     :query_event,               #  2
     :stop_event,                #  3
     :rotate_event,              #  4
     :intvar_event,              #  5
-    :load_event,                #  6
-    :slave_event,               #  7
-    :create_file_event,         #  8
+    :load_event,                #  6 (deprecated)
+    :slave_event,               #  7 (deprecated)
+    :create_file_event,         #  8 (deprecated)
     :append_block_event,        #  9
-    :exec_load_event,           # 10
+    :exec_load_event,           # 10 (deprecated)
     :delete_file_event,         # 11
-    :new_load_event,            # 12
+    :new_load_event,            # 12 (deprecated)
     :rand_event,                # 13
     :user_var_event,            # 14
     :format_description_event,  # 15
@@ -42,9 +23,9 @@ module MysqlBinlog
     :begin_load_query_event,    # 17
     :execute_load_query_event,  # 18
     :table_map_event,           # 19
-    :pre_ga_write_rows_event,   # 20
-    :pre_ga_update_rows_event,  # 21
-    :pre_ga_delete_rows_event,  # 22
+    :pre_ga_write_rows_event,   # 20 (deprecated)
+    :pre_ga_update_rows_event,  # 21 (deprecated)
+    :pre_ga_delete_rows_event,  # 22 (deprecated)
     :write_rows_event,          # 23
     :update_rows_event,         # 24
     :delete_rows_event,         # 25
@@ -52,38 +33,80 @@ module MysqlBinlog
     :heartbeat_log_event,       # 27
   ]
 
+  # Values for the +flags+ field that may appear in binary logs. There are
+  # several other values that never appear in a file but may be used
+  # in events in memory.
+  #
+  # Defined in sql/log_event.h line ~448
+  EVENT_HEADER_FLAGS = {
+    :binlog_in_use   => 0x01, # LOG_EVENT_BINLOG_IN_USE_F
+    :thread_specific => 0x04, # LOG_EVENT_THREAD_SPECIFIC_F
+    :suppress_use    => 0x08, # LOG_EVENT_SUPPRESS_USE_F
+    :artificial      => 0x20, # LOG_EVENT_ARTIFICIAL_F
+    :relay_log       => 0x40, # LOG_EVENT_RELAY_LOG_F
+  }
+
   # A mapping array for all values that may appear in the +status+ field of
   # a query_event.
+  #
+  # Defined in sql/log_event.h line ~316
   QUERY_EVENT_STATUS_TYPES = [
-    :flags2,                    # 0
-    :sql_mode,                  # 1
-    :catalog_deprecated,        # 2
-    :auto_increment,            # 3
-    :charset,                   # 4
-    :time_zone,                 # 5
-    :catalog,                   # 6
-    :lc_time_names,             # 7
-    :charset_database,          # 8
-    :table_map_for_update,      # 9
+    :flags2,                    #  0 (Q_FLAGS2_CODE)
+    :sql_mode,                  #  1 (Q_SQL_MODE_CODE)
+    :catalog_deprecated,        #  2 (Q_CATALOG_CODE)
+    :auto_increment,            #  3 (Q_AUTO_INCREMENT)
+    :charset,                   #  4 (Q_CHARSET_CODE)
+    :time_zone,                 #  5 (Q_TIME_ZONE_CODE)
+    :catalog,                   #  6 (Q_CATALOG_NZ_CODE)
+    :lc_time_names,             #  7 (Q_LC_TIME_NAMES_CODE)
+    :charset_database,          #  8 (Q_CHARSET_DATABASE_CODE)
+    :table_map_for_update,      #  9 (Q_TABLE_MAP_FOR_UPDATE_CODE)
+    :master_data_written,       # 10 (Q_MASTER_DATA_WRITTEN_CODE)
+    :invoker,                   # 11 (Q_INVOKER)
   ]
 
   # A mapping hash for all values that may appear in the +flags2+ field of
   # a query_event.
+  #
+  # Defined in sql/log_event.h line ~521 in OPTIONS_WRITTEN_TO_BIN_LOG
+  #
+  # Defined in sql/sql_priv.h line ~84
   QUERY_EVENT_FLAGS2 = {
-    :auto_is_null           => 1 << 14,
-    :not_autocommit         => 1 << 19,
-    :no_foreign_key_checks  => 1 << 26,
-    :relaxed_unique_checks  => 1 << 27,
+    :auto_is_null           => 1 << 14, # OPTION_AUTO_IS_NULL
+    :not_autocommit         => 1 << 19, # OPTION_NOT_AUTOCOMMIT
+    :no_foreign_key_checks  => 1 << 26, # OPTION_NO_FOREIGN_KEY_CHECKS
+    :relaxed_unique_checks  => 1 << 27, # OPTION_RELAXED_UNIQUE_CHECKS
   }
 
   # A mapping array for all values that may appear in the +Intvar_type+ field
   # of an intvar_event.
+  #
+  # Enumerated in sql/log_event.h line ~613 as Int_event_type
   INTVAR_EVENT_INTVAR_TYPES = [
-    nil,
-    :last_insert_id,
-    :insert_id,
+    nil,                # INVALID_INT_EVENT
+    :last_insert_id,    # LAST_INSERT_ID_EVENT
+    :insert_id,         # INSERT_ID_EVENT
   ]
 
+  # A mapping array for all values that may appear in the +flags+ field of a
+  # table_map_event.
+  #
+  # Enumerated in sql/log_event.h line ~3413 within Table_map_log_event
+  TABLE_MAP_EVENT_FLAGS = {
+    :bit_len_exact          => 1 << 0,  # TM_BIT_LEN_EXACT_F
+  }
+
+  # A mapping array for all values that may appear in the +flags+ field of a
+  # write_rows_event, update_rows_event, or delete_rows_event.
+  #
+  # Enumerated in sql/log_event.h line ~3533 within Rows_log_event
+  GENERIC_ROWS_EVENT_FLAGS = {
+    :stmt_end               => 1 << 0,  # STMT_END_F
+    :no_foreign_key_checks  => 1 << 1,  # NO_FOREIGN_KEY_CHECKS_F
+    :relaxed_unique_checks  => 1 << 2,  # RELAXED_UNIQUE_CHECKS_F
+    :complete_rows          => 1 << 3,  # COMPLETE_ROWS_F
+  }
+
   # Parse binary log events from a provided binary log. Must be driven
   # externally, but handles all the details of parsing an event header
   # and the content of the various event types.
@@ -105,26 +128,25 @@ module MysqlBinlog
       @table_map = {}
     end
 
-    # Parse an event header, described by the +EVENT_HEADER+ structure above.
+    # Parse an event header, which is consistent for all event types.
+    #
+    # Documented in sql/log_event.h line ~749 as "Common-Header"
+    #
+    # Implemented in sql/log_event.cc line ~936 in Log_event::write_header
     def event_header
-      header = parser.read_and_unpack(EVENT_HEADER)
-
-      # Merge the read +flags+ bitmap with the +EVENT_HEADER_FLAGS+ hash to
-      # return the flags by name instead of returning the bitmap as an integer.
-      flags = EVENT_HEADER_FLAGS.inject([]) do |result, (flag_name, flag_bit_value)|
-        if (header[:flags] & flag_bit_value) != 0
-          result << flag_name
-        end
-        result
-      end
-
-      # Overwrite the integer version of +flags+ with the array of names.
-      header[:flags] = flags
-
+      header = {}
+      header[:timestamp]      = parser.read_uint32
+      header[:event_type]     = EVENT_TYPES[parser.read_uint8]
+      header[:server_id]      = parser.read_uint32
+      header[:event_length]   = parser.read_uint32
+      header[:next_position]  = parser.read_uint32
+      header[:flags] = parser.read_uint_bitmap_by_size_and_name(2, EVENT_HEADER_FLAGS)
       header
     end
 
     # Parse fields for a +Format_description+ event.
+    #
+    # Implemented in sql/log_event.cc line ~4123 in Format_description_log_event::write
     def format_description_event(header)
       fields = {}
       fields[:binlog_version]   = parser.read_uint16
@@ -135,6 +157,8 @@ module MysqlBinlog
     end
 
     # Parse fields for a +Rotate+ event.
+    #
+    # Implemented in sql/log_event.cc line ~5157 in Rotate_log_event::write
     def rotate_event(header)
       fields = {}
       fields[:pos] = parser.read_uint64
@@ -157,7 +181,7 @@ module MysqlBinlog
         status_type = QUERY_EVENT_STATUS_TYPES[parser.read_uint8]
         status[status_type] = case status_type
         when :flags2
-          parser.read_uint32_bitmap_by_name(QUERY_EVENT_FLAGS2)
+          parser.read_uint_bitmap_by_size_and_name(4, QUERY_EVENT_FLAGS2)
         when :sql_mode
           parser.read_uint64
         when :catalog_deprecated
@@ -185,11 +209,21 @@ module MysqlBinlog
           parser.read_uint64
         end
       end
+
+      # We may have read too much due to an invalid string read especially.
+      # Raise a more specific exception here instead of the generic
+      # OverReadException from the entire event.
+      if reader.position > end_position
+        raise OverReadException.new("Read past end of Query event status field")
+      end
+
       status
     end
     private :_query_event_status
 
     # Parse fields for a +Query+ event.
+    #
+    # Implemented in sql/log_event.cc line ~2214 in Query_log_event::write
     def query_event(header)
       fields = {}
       fields[:thread_id] = parser.read_uint32
@@ -204,6 +238,8 @@ module MysqlBinlog
     end
 
     # Parse fields for an +Intvar+ event.
+    #
+    # Implemented in sql/log_event.cc line ~5326 in Intvar_log_event::write
     def intvar_event(header)
       fields = {}
 
@@ -215,6 +251,8 @@ module MysqlBinlog
     end
 
     # Parse fields for an +Xid+ event.
+    #
+    # Implemented in sql/log_event.cc line ~5559 in Xid_log_event::write
     def xid_event(header)
       fields = {}
       fields[:xid] = parser.read_uint64
@@ -222,6 +260,8 @@ module MysqlBinlog
     end
 
     # Parse fields for an +Rand+ event.
+    #
+    # Implemented in sql/log_event.cc line ~5454 in Rand_log_event::write
     def rand_event(header)
       fields = {}
       fields[:seed1] = parser.read_uint64
@@ -239,9 +279,10 @@ module MysqlBinlog
       when :varchar
         { :max_length => parser.read_uint16 }
       when :bit
+        bits  = parser.read_uint8
+        bytes = parser.read_uint8
         {
-          :size_bits  => parser.read_uint8,
-          :size_bytes => parser.read_uint8,
+          :bits  => (bytes * 8) + bits
         }
       when :newdecimal
         {
@@ -276,10 +317,14 @@ module MysqlBinlog
     private :_table_map_event_column_metadata
 
     # Parse fields for a +Table_map+ event.
+    #
+    # Implemented in sql/log_event.cc line ~8638
+    # in Table_map_log_event::write_data_header
+    # and Table_map_log_event::write_data_body
     def table_map_event(header)
       fields = {}
       fields[:table_id] = parser.read_uint48
-      fields[:flags] = parser.read_uint16
+      fields[:flags] = parser.read_uint_bitmap_by_size_and_name(2, TABLE_MAP_EVENT_FLAGS)
       map_entry = @table_map[fields[:table_id]] = {}
       map_entry[:db] = parser.read_lpstringz
       map_entry[:table] = parser.read_lpstringz
@@ -337,7 +382,7 @@ module MysqlBinlog
       end_position = reader.position + reader.remaining(header)
       while reader.position < end_position
         row_image = {}
-        case EVENT_TYPES[header[:event_type]]
+        case header[:event_type]
         when :write_rows_event
           row_image[:after]  = _generic_rows_event_row_image(header, fields, columns_used[:after])
         when :delete_rows_event
@@ -348,6 +393,14 @@ module MysqlBinlog
         end
         row_images << row_image
       end
+
+      # We may have read too much, especially if any of the fields in the row
+      # image were misunderstood. Raise a more specific exception here instead
+      # of the generic OverReadException from the entire event.
+      if reader.position > end_position
+        raise OverReadException.new("Read past end of row image")
+      end
+
       row_images
     end
     private :_generic_rows_event_row_images
@@ -356,14 +409,18 @@ module MysqlBinlog
     # * +Write_rows+ which is used for +INSERT+.
     # * +Update_rows+ which is used for +UPDATE+.
     # * +Delete_rows+ which is used for +DELETE+.
+    #
+    # Implemented in sql/log_event.cc line ~8039
+    # in Rows_log_event::write_data_header
+    # and Rows_log_event::write_data_body
     def generic_rows_event(header)
       fields = {}
       table_id = parser.read_uint48
       fields[:table] = @table_map[table_id]
-      fields[:flags] = parser.read_uint16
+      fields[:flags] = parser.read_uint_bitmap_by_size_and_name(2, GENERIC_ROWS_EVENT_FLAGS)
       columns = parser.read_varint
       columns_used = {}
-      case EVENT_TYPES[header[:event_type]]
+      case header[:event_type]
       when :write_rows_event
         columns_used[:after]  = parser.read_bit_array(columns)
       when :delete_rows_event

data/lib/mysql_binlog/binlog_field_parser.rb

@@ -99,12 +99,12 @@ module MysqlBinlog
 
     # Read a single-precision (4-byte) floating point number.
     def read_float
-      reader.read(4).unpack("g").first
+      reader.read(4).unpack("e").first
     end
 
     # Read a double-precision (8-byte) floating point number.
     def read_double
-      reader.read(8).unpack("G").first
+      reader.read(8).unpack("E").first
     end
 
     # Read a variable-length "Length Coded Binary" integer. This is derived
@@ -174,16 +174,18 @@ module MysqlBinlog
     end
 
     # Read an arbitrary-length bitmap, provided its length. Returns an array
-    # of true/false values.
+    # of true/false values. This is used both for internal usage in RBR
+    # events that need bitmaps, as well as for the BIT type.
     def read_bit_array(length)
       data = reader.read((length+7)/8)
-      data.unpack("b*").first.bytes.to_a.map { |i| (i-48) == 1 }.shift(length)
+      data.unpack("B*").first.reverse.  # Unpack into a string of "10101"
+        split("").map { |c| c == "1" }.shift(length) # Return true/false array
     end
 
-    # Read a uint32 value, and convert it to an array of symbols derived
-    # from a mapping table provided.
-    def read_uint32_bitmap_by_name(names)
-      value = read_uint32
+    # Read a uint value using the provided size, and convert it to an array
+    # of symbols derived from a mapping table provided.
+    def read_uint_bitmap_by_size_and_name(size, names)
+      value = read_uint_by_size(size)
       names.inject([]) do |result, (name, bit_value)|
         if (value & bit_value) != 0
           result << name
@@ -192,25 +194,6 @@ module MysqlBinlog
       end
     end
 
-    # Read a series of fields, provided an array of field descriptions. This
-    # can be used to read many types of fixed-length structures.
-    def read_and_unpack(format_description)
-      @format_cache[format_description] ||= {}
-      this_format = @format_cache[format_description][:format] ||= 
-        format_description.inject("") { |o, f| o+(f[:format] || "") }
-      this_length = @format_cache[format_description][:length] ||=
-        format_description.inject(0)  { |o, f| o+(f[:length] || 0) }
-
-      fields = {}
-
-      fields_array = reader.read(this_length).unpack(this_format)
-      format_description.each_with_index do |field, index| 
-        fields[field[:name]] = fields_array[index]
-      end
-
-      fields
-    end
-
     # Extract a number of sequential bits at a given offset within an integer.
     # This is used to unpack bit-packed fields.
     def extract_bits(value, bits, offset)
@@ -289,7 +272,8 @@ module MysqlBinlog
         convert_mysql_type_datetime(read_uint64)
       when :enum, :set
         read_uint_by_size(metadata[:size])
-      #when :bit
+      when :bit
+        read_bit_array(metadata[:bits])
       #when :newdecimal
       end
     end

data/lib/mysql_binlog/reader/binlog_file_reader.rb

@@ -1,6 +1,9 @@
 module MysqlBinlog
   # Read a binary log from a file on disk.
   class BinlogFileReader
+    MAGIC_SIZE  = 4
+    MAGIC_VALUE = 1852400382
+
     attr_accessor :tail
 
     def initialize(filename)
@@ -8,14 +11,18 @@ module MysqlBinlog
       open_file(filename)
     end
 
+    def verify_magic
+      if (magic = read(MAGIC_SIZE).unpack("V").first) != MAGIC_VALUE
+        raise MalformedBinlogException.new("Magic number #{magic} is incorrect")
+      end
+    end
+
     def open_file(filename)
       @dirname  = File.dirname(filename)
       @filename = File.basename(filename)
       @binlog   = File.open(filename, mode="r")
 
-      if (magic = read(4).unpack("V").first) != 1852400382
-        raise MalformedBinlogException.new("Magic number #{magic} is incorrect")
-      end
+      verify_magic
     end
 
     def rotate(filename, position)
@@ -24,6 +31,9 @@ module MysqlBinlog
         open_file(@dirname + "/" + filename)
         seek(position)
       rescue Errno::ENOENT
+        # A rotate event will be seen in the previous log file before the
+        # new file exists. Retry a few times with a little sleep to give
+        # the server a chance to create the new file.
         if (retries -= 1) > 0
           sleep 0.01
           retry
@@ -42,7 +52,7 @@ module MysqlBinlog
     end
 
     def rewind
-      @binlog.rewind
+      seek(MAGIC_SIZE)
     end
 
     def seek(pos)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mysql_binlog
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 23
   prerelease: 
   segments: 
   - 0
-  - 1
-  - 8
-  version: 0.1.8
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Jeremy Cole
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-02 00:00:00 Z
+date: 2012-07-05 00:00:00 Z
 dependencies: []
 
 description: Library for parsing MySQL binary logs in Ruby