mysql_binlog 0.1.5 → 0.1.6

data/lib/mysql_binlog.rb

@@ -4,3 +4,8 @@ require 'mysql_binlog/binlog_event_parser'
 require 'mysql_binlog/reader/debugging_reader'
 require 'mysql_binlog/reader/binlog_file_reader'
 require 'mysql_binlog/reader/binlog_stream_reader'
+
+# The MysqlBinlog module contains a series of classes for reading and
+# parsing binary log events from MySQL binary logs.
+module MysqlBinlog
+end

data/lib/mysql_binlog/binlog.rb

@@ -1,9 +1,37 @@
 module MysqlBinlog
+  # This version of the binary log format is not supported by this library.
   class UnsupportedVersionException < Exception; end
+
+  # An error was encountered when trying to read the log, which was likely
+  # due to garbage data in the log. Continuing is likely impossible.
   class MalformedBinlogException < Exception; end
+
+  # When attempting a read, no data was returned.
   class ZeroReadException < Exception; end
+
+  # When attempting a read, fewer bytes of data were returned than were
+  # requested by the reader, likely indicating a truncated file or corrupted
+  # event.
   class ShortReadException < Exception; end
 
+  # Read a binary log, parsing and returning events.
+  #
+  # == Examples
+  #
+  # A basic example of using the Binlog class:
+  #
+  #   require 'mysql_binlog'
+  #   include MysqlBinlog
+  #
+  #   # Open a binary log from a file on disk.
+  #   binlog = Binlog.new(BinlogFileReader.new("mysql-bin.000001"))
+  #
+  #   # Iterate over all events from the log, printing the event type (such
+  #   # as :query_event, :write_rows_event, etc.)
+  #   binlog.each_event do |event|
+  #     puts event[:type]
+  #   end
+  #
   class Binlog
     attr_reader :fde
     attr_accessor :reader
@@ -35,6 +63,7 @@ module MysqlBinlog
     def skip_event(header)
       reader.skip(header)
     end
+    private :skip_event
 
     # Read the content of the event, which follows the header.
     def read_event_fields(header)
@@ -53,6 +82,7 @@ module MysqlBinlog
 
       fields
     end
+    private :read_event_fields
 
     # Scan events until finding one that isn't rejected by the filter rules.
     # If there are no filter rules, this will return the next event provided
@@ -130,6 +160,7 @@ module MysqlBinlog
         :server_version => fde[:server_version],
       }
     end
+    private :process_fde
 
     # Iterate through all events.
     def each_event

data/lib/mysql_binlog/binlog_event_parser.rb

@@ -9,7 +9,7 @@ module MysqlBinlog
     { :name => :flags,            :length => 2,   :format => "v"   },
   ]
 
-  # Values for the 'flags' field that may appear in binlogs. There are
+  # Values for the +flags+ field that may appear in binary logs. There are
   # several other values that never appear in a file but may be used
   # in events in memory.
   EVENT_HEADER_FLAGS = {
@@ -76,15 +76,26 @@ module MysqlBinlog
     :relaxed_unique_checks  => 1 << 27,
   }
 
+  # A mapping array for all values that may appear in the +Intvar_type+ field
+  # of an intvar_event.
   INTVAR_EVENT_INTVAR_TYPES = [
     nil,
     :last_insert_id,
     :insert_id,
   ]
 
+  # Parse binary log events from a provided binary log. Must be driven
+  # externally, but handles all the details of parsing an event header
+  # and the content of the various event types.
   class BinlogEventParser
+    # The binary log object this event parser will parse events from.
     attr_accessor :binlog
+
+    # The binary log reader extracted from the binlog object for convenience.
     attr_accessor :reader
+
+    # The binary log field parser extracted from the binlog object for
+    # convenience.
     attr_accessor :parser
 
     def initialize(binlog_instance)
@@ -94,12 +105,12 @@ module MysqlBinlog
       @table_map = {}
     end
 
-    # Parse an event header, described by the EVENT_HEADER structure above.
+    # Parse an event header, described by the +EVENT_HEADER+ structure above.
     def event_header
       header = parser.read_and_unpack(EVENT_HEADER)
 
-      # Merge the read 'flags' bitmap with the EVENT_HEADER_FLAGS hash to return
-      # the flags by name instead of returning the bitmap as an integer.
+      # Merge the read +flags+ bitmap with the +EVENT_HEADER_FLAGS+ hash to
+      # return the flags by name instead of returning the bitmap as an integer.
       flags = EVENT_HEADER_FLAGS.inject([]) do |result, (flag_name, flag_bit_value)|
         if (header[:flags] & flag_bit_value) != 0
           result << flag_name
@@ -107,7 +118,7 @@ module MysqlBinlog
         result
       end
 
-      # Overwrite the integer version of 'flags' with the array of names.
+      # Overwrite the integer version of +flags+ with the array of names.
       header[:flags] = flags
 
       header
@@ -135,7 +146,7 @@ module MysqlBinlog
     # Parse a dynamic +status+ structure within a query_event, which consists
     # of a status_length (uint16) followed by a number of status variables
     # (determined by the +status_length+) each of which consist of:
-    # * A type code (uint8), one of QUERY_EVENT_STATUS_TYPES.
+    # * A type code (+uint8+), one of +QUERY_EVENT_STATUS_TYPES+.
     # * The content itself, determined by the type. Additional processing is
     #   required based on the type.
     def _query_event_status(header, fields)
@@ -176,6 +187,7 @@ module MysqlBinlog
       end
       status
     end
+    private :_query_event_status
 
     # Parse fields for a +Query+ event.
     def query_event(header)
@@ -217,13 +229,44 @@ module MysqlBinlog
       fields
     end
 
-    # Parse column metadata within a table map event.
+    # Parse a number of bytes from the metadata section of a +Table_map+ event
+    # representing various fields based on the column type of the column
+    # being processed.
+    def _table_map_event_column_metadata_read(column_type)
+      case column_type
+      when :float, :double
+        { :size => parser.read_uint8 }
+      when :varchar
+        { :max_length => parser.read_uint16 }
+      when :bit
+        {
+          :size_bits  => parser.read_uint8,
+          :size_bytes => parser.read_uint8,
+        }
+      when :newdecimal
+        {
+          :precision => parser.read_uint8,
+          :decimals  => parser.read_uint8,
+        }
+      when :blob, :geometry
+        { :length_size => parser.read_uint8 }
+      when :string, :var_string
+        {
+          :real_type   => MYSQL_TYPES[parser.read_uint8],
+          :max_length  => parser.read_uint8,
+        }
+      end
+    end
+    private :_table_map_event_column_metadata_read
+
+    # Parse column metadata within a +Table_map+ event.
     def _table_map_event_column_metadata(columns_type)
       length = parser.read_varint
-      columns_type.map do |c|
-        parser.read_mysql_type_metadata(c)
+      columns_type.map do |column|
+        _table_map_event_column_metadata_read(column)
       end
     end
+    private :_table_map_event_column_metadata
 
     # Parse fields for a +Table_map+ event.
     def table_map_event(header)
@@ -269,6 +312,7 @@ module MysqlBinlog
       end
       row_image
     end
+    private :_generic_rows_event_row_image
 
     # Parse the row images present in a row-based replication row event. This
     # is rather incomplete right now due missing support for many MySQL types,
@@ -291,6 +335,7 @@ module MysqlBinlog
       end
       row_images
     end
+    private :_generic_rows_event_row_images
 
     # Parse fields for any of the row-based replication row events:
     # * +Write_rows+ which is used for +INSERT+.

data/lib/mysql_binlog/binlog_field_parser.rb

@@ -36,6 +36,8 @@ module MysqlBinlog
    type_array
   end
 
+  # Parse various types of standard and non-standard data types from a
+  # provided binary log using its reader to read data.
   class BinlogFieldParser
     attr_accessor :binlog
     attr_accessor :reader
@@ -201,29 +203,46 @@ module MysqlBinlog
       fields
     end
 
-    def read_mysql_type_metadata(column_type)
-      case column_type
-      when :float, :double
-        { :size => read_uint8 }
-      when :varchar
-        { :max_length => read_uint16 }
-      when :bit
-        { :size => read_uint8 }
-      when :decimal
-        {
-          :precision => read_uint8,
-          :decimals  => read_uint8,
-        }
-      when :blob
-        { :length_size => read_uint8 }
-      when :string, :var_string
-        {
-          :real_type   => read_uint8,
-          :max_length  => read_uint8,
-        }
-      when :geometry
-        { :length_size => read_uint8 }
-      end
+    # Extract a number of sequential bits at a given offset within an integer.
+    # This is used to unpack bit-packed fields.
+    def extract_bits(value, bits, offset)
+      (value & ((1 << bits) - 1) << offset) >> offset
+    end
+
+    # Convert a packed +DATE+ from a uint24 into a string representing
+    # the date.
+    def convert_mysql_type_date(value)
+      "%04i-%02i-%02i" % [
+        extract_bits(value, 15, 9),
+        extract_bits(value,  4, 5),
+        extract_bits(value,  5, 0),
+      ]
+    end
+
+    # Convert a packed +TIME+ from a uint24 into a string representing
+    # the time.
+    def convert_mysql_type_time(value)
+      "%02i:%02i:%02i" % [
+        value / 10000,
+        (value % 10000) / 100,
+        value % 100,
+      ]
+    end
+
+    # Convert a packed +DATETIME+ from a uint64 into a string representing
+    # the date and time.
+    def convert_mysql_type_datetime(value)
+      date = value / 1000000
+      time = value % 1000000
+
+      "%04i-%02i-%02i %02i:%02i:%02i" % [
+        date / 10000,
+        (date % 10000) / 100,
+        date % 100,
+        time / 10000,
+        (time % 10000) / 100,
+        time % 100,
+      ]
     end
 
     # Read a single field, provided the MySQL column type as a symbol. Not all
@@ -248,24 +267,21 @@ module MysqlBinlog
         read_varstring
       when :varchar
         read_lpstring(2)
-      when :blob
+      when :blob, :geometry
         read_lpstring(metadata[:length_size])
       when :timestamp
         read_uint32
       when :year
         read_uint8 + 1900
-      #when :date
-      #when :time
-      #when :datetime
-      #when :newdate
+      when :date
+        convert_mysql_type_date(read_uint24)
+      when :time
+        convert_mysql_type_time(read_uint24)
+      when :datetime
+        convert_mysql_type_datetime(read_uint64)
       #when :bit
-      #when :decimal
       #when :newdecimal
-      #when :enum
-      #when :set
-      #when :geometry
       end
     end
-
   end
 end

data/lib/mysql_binlog/reader/binlog_file_reader.rb

@@ -1,4 +1,5 @@
 module MysqlBinlog
+  # Read a binary log from a file on disk.
   class BinlogFileReader
     def initialize(filename)
       @filename = filename

data/lib/mysql_binlog/reader/binlog_stream_reader.rb

@@ -1,4 +1,7 @@
 module MysqlBinlog
+  # Read a binary log from a stream dumped using the +MysqlBinlogDump+
+  # library to request a +COM_BINLOG_DUMP+ from a MySQL server via the
+  # +Mysql+ library.
   class BinlogStreamReader
     def initialize(connection, filename, position)
       require 'mysql_binlog_dump'
@@ -11,7 +14,6 @@ module MysqlBinlog
     end
 
     def rotate(filename, position)
-      puts "rotate called with #{filename}:#{position}"
       @filename = filename
       @position = position
     end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mysql_binlog
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 23
   prerelease: 
   segments: 
   - 0
   - 1
-  - 5
-  version: 0.1.5
+  - 6
+  version: 0.1.6
 platform: ruby
 authors: 
 - Jeremy Cole