mysql_binlog 0.1.4 → 0.1.5

data/lib/mysql_binlog.rb

@@ -1,6 +1,6 @@
-require 'mysql_binlog/mysql_binlog'
-require 'mysql_binlog/binlog_parser'
-require 'mysql_binlog/binlog_event_field_parser'
+require 'mysql_binlog/binlog'
+require 'mysql_binlog/binlog_field_parser'
+require 'mysql_binlog/binlog_event_parser'
 require 'mysql_binlog/reader/debugging_reader'
 require 'mysql_binlog/reader/binlog_file_reader'
 require 'mysql_binlog/reader/binlog_stream_reader'

data/lib/mysql_binlog/binlog.rb

@@ -0,0 +1,141 @@
+module MysqlBinlog
+  class UnsupportedVersionException < Exception; end
+  class MalformedBinlogException < Exception; end
+  class ZeroReadException < Exception; end
+  class ShortReadException < Exception; end
+
+  class Binlog
+    attr_reader :fde
+    attr_accessor :reader
+    attr_accessor :field_parser
+    attr_accessor :event_parser
+    attr_accessor :filter_event_types
+    attr_accessor :filter_flags
+    attr_accessor :max_query_length
+
+    def initialize(reader)
+      @reader = reader
+      @field_parser = BinlogFieldParser.new(self)
+      @event_parser = BinlogEventParser.new(self)
+      @fde = nil
+      @filter_event_types = nil
+      @filter_flags = nil
+      @max_query_length = 1048576
+    end
+
+    # Rewind to the beginning of the log, if supported by the reader. The
+    # reader may throw an exception if rewinding is not supported (e.g. for
+    # a stream-based reader).
+    def rewind
+      reader.rewind
+    end
+
+    # Skip the remainder of this event. This can be used to skip an entire
+    # event or merely the parts of the event this library does not understand.
+    def skip_event(header)
+      reader.skip(header)
+    end
+
+    # Read the content of the event, which follows the header.
+    def read_event_fields(header)
+      event_type = EVENT_TYPES[header[:event_type]]
+
+      # Delegate the parsing of the event content to a method of the same name
+      # in BinlogEventParser.
+      if event_parser.methods.include? event_type.to_s
+        fields = event_parser.send(event_type, header)
+      end
+
+      # Anything left unread at this point is skipped based on the event length
+      # provided in the header. In this way, it is possible to skip over events
+      # that are not able to be parsed correctly by this library.
+      skip_event(header)
+
+      fields
+    end
+
+    # Scan events until finding one that isn't rejected by the filter rules.
+    # If there are no filter rules, this will return the next event provided
+    # by the reader.
+    def read_event
+      while true
+        skip_this_event = false
+        return nil if reader.end?
+
+        filename = reader.filename
+        position = reader.position
+
+        # Read the common header for an event. Every event has a header.
+        unless header = event_parser.event_header
+          return nil
+        end
+
+        event_type = EVENT_TYPES[header[:event_type]]
+        
+        if @filter_event_types
+          unless @filter_event_types.include? event_type
+            skip_this_event = true
+          end
+        end
+        
+        if @filter_flags
+          unless @filter_flags.include? header[:flags]
+            skip_this_event = true
+          end
+        end
+
+        # Never skip over rotate_event or format_description_event as they
+        # are critical to understanding the format of this event stream.
+        if skip_this_event
+          unless [:rotate_event, :format_description_event].include? event_type
+            skip_event(header)
+            next
+          end
+        end
+        
+        fields = read_event_fields(header)
+
+        case event_type
+        when :rotate_event
+          reader.rotate(fields[:name], fields[:pos])
+        when :format_description_event
+          process_fde(fields)
+        end
+
+        break
+      end
+
+      {
+        :type     => event_type,
+        :filename => filename,
+        :position => position,
+        :header   => header,
+        :event    => fields,
+      }
+    end
+
+    # Process a format description event, which describes the version of this
+    # file, and the format of events which will appear in this file. This also
+    # provides the version of the MySQL server which generated this file.
+    def process_fde(fde)
+      if (version = fde[:binlog_version]) != 4
+        raise UnsupportedVersionException.new("Binlog version #{version} is not supported")
+      end
+
+      # Save the interesting fields from an FDE so that this information is
+      # available at any time later.
+      @fde = {
+        :header_length  => fde[:header_length],
+        :binlog_version => fde[:binlog_version],
+        :server_version => fde[:server_version],
+      }
+    end
+
+    # Iterate through all events.
+    def each_event
+      while event = read_event
+        yield event
+      end
+    end
+  end
+end

data/lib/mysql_binlog/{binlog_event_field_parser.rb → binlog_event_parser.rb}

@@ -1,4 +1,57 @@
 module MysqlBinlog
+  # A common fixed-length header that is included with each event.
+  EVENT_HEADER = [
+    { :name => :timestamp,        :length => 4,   :format => "V"   },
+    { :name => :event_type,       :length => 1,   :format => "C"   },
+    { :name => :server_id,        :length => 4,   :format => "V"   },
+    { :name => :event_length,     :length => 4,   :format => "V"   },
+    { :name => :next_position,    :length => 4,   :format => "V"   },
+    { :name => :flags,            :length => 2,   :format => "v"   },
+  ]
+
+  # Values for the 'flags' field that may appear in binlogs. There are
+  # several other values that never appear in a file but may be used
+  # in events in memory.
+  EVENT_HEADER_FLAGS = {
+    :binlog_in_use   => 0x01,
+    :thread_specific => 0x04,
+    :suppress_use    => 0x08,
+    :artificial      => 0x20,
+    :relay_log       => 0x40,
+  }
+
+  # An array to quickly map an integer event type to its symbol.
+  EVENT_TYPES = [
+    :unknown_event,             #  0
+    :start_event_v3,            #  1
+    :query_event,               #  2
+    :stop_event,                #  3
+    :rotate_event,              #  4
+    :intvar_event,              #  5
+    :load_event,                #  6
+    :slave_event,               #  7
+    :create_file_event,         #  8
+    :append_block_event,        #  9
+    :exec_load_event,           # 10
+    :delete_file_event,         # 11
+    :new_load_event,            # 12
+    :rand_event,                # 13
+    :user_var_event,            # 14
+    :format_description_event,  # 15
+    :xid_event,                 # 16
+    :begin_load_query_event,    # 17
+    :execute_load_query_event,  # 18
+    :table_map_event,           # 19
+    :pre_ga_write_rows_event,   # 20
+    :pre_ga_update_rows_event,  # 21
+    :pre_ga_delete_rows_event,  # 22
+    :write_rows_event,          # 23
+    :update_rows_event,         # 24
+    :delete_rows_event,         # 25
+    :incident_event,            # 26
+    :heartbeat_log_event,       # 27
+  ]
+
   # A mapping array for all values that may appear in the +status+ field of
   # a query_event.
   QUERY_EVENT_STATUS_TYPES = [
@@ -23,7 +76,13 @@ module MysqlBinlog
     :relaxed_unique_checks  => 1 << 27,
   }
 
-  class BinlogEventFieldParser
+  INTVAR_EVENT_INTVAR_TYPES = [
+    nil,
+    :last_insert_id,
+    :insert_id,
+  ]
+
+  class BinlogEventParser
     attr_accessor :binlog
     attr_accessor :reader
     attr_accessor :parser
@@ -31,14 +90,46 @@ module MysqlBinlog
     def initialize(binlog_instance)
       @binlog = binlog_instance
       @reader = binlog_instance.reader
-      @parser = binlog_instance.parser
+      @parser = binlog_instance.field_parser
       @table_map = {}
     end
 
-    # Parse additional fields for a +Rotate+ event.
-    def rotate_event(header, fields)
+    # Parse an event header, described by the EVENT_HEADER structure above.
+    def event_header
+      header = parser.read_and_unpack(EVENT_HEADER)
+
+      # Merge the read 'flags' bitmap with the EVENT_HEADER_FLAGS hash to return
+      # the flags by name instead of returning the bitmap as an integer.
+      flags = EVENT_HEADER_FLAGS.inject([]) do |result, (flag_name, flag_bit_value)|
+        if (header[:flags] & flag_bit_value) != 0
+          result << flag_name
+        end
+        result
+      end
+
+      # Overwrite the integer version of 'flags' with the array of names.
+      header[:flags] = flags
+
+      header
+    end
+
+    # Parse fields for a +Format_description+ event.
+    def format_description_event(header)
+      fields = {}
+      fields[:binlog_version]   = parser.read_uint16
+      fields[:server_version]   = parser.read_nstringz(50)
+      fields[:create_timestamp] = parser.read_uint32
+      fields[:header_length]    = parser.read_uint8
+      fields
+    end
+
+    # Parse fields for a +Rotate+ event.
+    def rotate_event(header)
+      fields = {}
+      fields[:pos] = parser.read_uint64
       name_length = reader.remaining(header)
-      fields[:name] = reader.read(name_length)
+      fields[:name] = parser.read_nstring(name_length)
+      fields
     end
 
     # Parse a dynamic +status+ structure within a query_event, which consists
@@ -86,25 +177,44 @@ module MysqlBinlog
       status
     end
 
-    # Parse additional fields for a +Query+ event.
-    def query_event(header, fields)
+    # Parse fields for a +Query+ event.
+    def query_event(header)
+      fields = {}
+      fields[:thread_id] = parser.read_uint32
+      fields[:elapsed_time] = parser.read_uint32
+      db_length = parser.read_uint8
+      fields[:error_code] = parser.read_uint16
       fields[:status] = _query_event_status(header, fields)
-      fields[:db] = parser.read_nstringz(fields[:db_length])
-      fields.delete :db_length
+      fields[:db] = parser.read_nstringz(db_length + 1)
       query_length = reader.remaining(header)
       fields[:query] = reader.read([query_length, binlog.max_query_length].min)
+      fields
     end
 
-    # Parse additional fields for an +Intvar+ event.
-    def intvar_event(header, fields)
-      case fields[:intvar_type]
-      when 1
-        fields[:intvar_name] = :last_insert_id
-      when 2
-        fields[:intvar_name] = :insert_id
-      else
-        fields[:intvar_name] = nil
-      end
+    # Parse fields for an +Intvar+ event.
+    def intvar_event(header)
+      fields = {}
+
+      fields[:intvar_type]  = parser.read_uint8
+      fields[:intvar_name]  = INTVAR_EVENT_INTVAR_TYPES[fields[:intvar_type]]
+      fields[:intvar_value] = parser.read_uint64
+
+      fields
+    end
+
+    # Parse fields for an +Xid+ event.
+    def xid_event(header)
+      fields = {}
+      fields[:xid] = parser.read_uint64
+      fields
+    end
+
+    # Parse fields for an +Rand+ event.
+    def rand_event(header)
+      fields = {}
+      fields[:seed1] = parser.read_uint64
+      fields[:seed2] = parser.read_uint64
+      fields
     end
 
     # Parse column metadata within a table map event.
@@ -115,8 +225,9 @@ module MysqlBinlog
       end
     end
 
-    # Parse additional fields for a +Table_map+ event.
-    def table_map_event(header, fields)
+    # Parse fields for a +Table_map+ event.
+    def table_map_event(header)
+      fields = {}
       fields[:table_id] = parser.read_uint48
       fields[:flags] = parser.read_uint16
       map_entry = @table_map[fields[:table_id]] = {}
@@ -136,6 +247,7 @@ module MysqlBinlog
       end
 
       fields[:map_entry] = map_entry
+      fields
     end
 
     # Parse a single row image, which is comprised of a series of columns. Not
@@ -180,11 +292,12 @@ module MysqlBinlog
       row_images
     end
 
-    # Parse additional fields for any of the row-based replication row events:
+    # Parse fields for any of the row-based replication row events:
     # * +Write_rows+ which is used for +INSERT+.
     # * +Update_rows+ which is used for +UPDATE+.
     # * +Delete_rows+ which is used for +DELETE+.
-    def generic_rows_event(header, fields)
+    def generic_rows_event(header)
+      fields = {}
       table_id = parser.read_uint48
       fields[:table] = @table_map[table_id]
       fields[:flags] = parser.read_uint16
@@ -200,7 +313,9 @@ module MysqlBinlog
         columns_used[:after]  = parser.read_bit_array(columns)
       end
       fields[:row_image] = _generic_rows_event_row_images(header, fields, columns_used)
+      fields
     end
+
     alias :write_rows_event  :generic_rows_event
     alias :update_rows_event :generic_rows_event
     alias :delete_rows_event :generic_rows_event

data/lib/mysql_binlog/{binlog_parser.rb → binlog_field_parser.rb}

@@ -36,7 +36,7 @@ module MysqlBinlog
    type_array
   end
 
-  class BinlogParser
+  class BinlogFieldParser
     attr_accessor :binlog
     attr_accessor :reader
 
@@ -120,11 +120,9 @@ module MysqlBinlog
       reader.read(length)
     end
 
-    # Read a null-terminated string, provided its length (without the null).
+    # Read a null-terminated string, provided its length (with the null).
     def read_nstringz(length)
-      string = read_nstring(length)
-      reader.read(1) # null
-      string
+      reader.read(length).unpack("A*").first
     end
 
     # Read a (Pascal-style) length-prefixed string. The length is stored as a

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mysql_binlog
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 0
   - 1
-  - 4
-  version: 0.1.4
+  - 5
+  version: 0.1.5
 platform: ruby
 authors: 
 - Jeremy Cole
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-25 00:00:00 Z
+date: 2012-06-28 00:00:00 Z
 dependencies: []
 
 description: Library for parsing MySQL binary logs in Ruby
@@ -28,9 +28,9 @@ extra_rdoc_files: []
 
 files: 
 - lib/mysql_binlog.rb
-- lib/mysql_binlog/mysql_binlog.rb
-- lib/mysql_binlog/binlog_parser.rb
-- lib/mysql_binlog/binlog_event_field_parser.rb
+- lib/mysql_binlog/binlog.rb
+- lib/mysql_binlog/binlog_field_parser.rb
+- lib/mysql_binlog/binlog_event_parser.rb
 - lib/mysql_binlog/reader/binlog_file_reader.rb
 - lib/mysql_binlog/reader/binlog_stream_reader.rb
 - bin/mysql_binlog_dump

data/lib/mysql_binlog/mysql_binlog.rb

@@ -1,262 +0,0 @@
-module MysqlBinlog
-  # An array to quickly map an integer event type to its symbol.
-  EVENT_TYPES = [
-    :unknown_event,             #  0
-    :start_event_v3,            #  1
-    :query_event,               #  2
-    :stop_event,                #  3
-    :rotate_event,              #  4
-    :intvar_event,              #  5
-    :load_event,                #  6
-    :slave_event,               #  7
-    :create_file_event,         #  8
-    :append_block_event,        #  9
-    :exec_load_event,           # 10
-    :delete_file_event,         # 11
-    :new_load_event,            # 12
-    :rand_event,                # 13
-    :user_var_event,            # 14
-    :format_description_event,  # 15
-    :xid_event,                 # 16
-    :begin_load_query_event,    # 17
-    :execute_load_query_event,  # 18
-    :table_map_event,           # 19
-    :pre_ga_write_rows_event,   # 20
-    :pre_ga_update_rows_event,  # 21
-    :pre_ga_delete_rows_event,  # 22
-    :write_rows_event,          # 23
-    :update_rows_event,         # 24
-    :delete_rows_event,         # 25
-    :incident_event,            # 26
-    :heartbeat_log_event,       # 27
-  ]
-
-  # A common fixed-length header that is included with each event.
-  EVENT_HEADER = [
-    { :name => :timestamp,        :length => 4,   :format => "V"   },
-    { :name => :event_type,       :length => 1,   :format => "C"   },
-    { :name => :server_id,        :length => 4,   :format => "V"   },
-    { :name => :event_length,     :length => 4,   :format => "V"   },
-    { :name => :next_position,    :length => 4,   :format => "V"   },
-    { :name => :flags,            :length => 2,   :format => "v"   },
-  ]
-
-  # Values for the 'flags' field that may appear in binlogs. There are
-  # several other values that never appear in a file but may be used
-  # in events in memory.
-  EVENT_FLAGS = {
-    :binlog_in_use   => 0x01,
-    :thread_specific => 0x04,
-    :suppress_use    => 0x08,
-    :artificial      => 0x20,
-    :relay_log       => 0x40,
-  }
-
-  # Format descriptions for fixed-length fields that may appear in the data
-  # for each event. Additional fields may be dynamic and are parsed by the
-  # methods in the BinlogEventFieldParser class.
-  EVENT_FORMATS = {
-    :format_description_event => [
-      { :name => :binlog_version,   :length => 2,   :format => "v"   },
-      { :name => :server_version,   :length => 50,  :format => "A50" },
-      { :name => :create_timestamp, :length => 4,   :format => "V"   },
-      { :name => :header_length,    :length => 1,   :format => "C"   },
-    ],
-    :rotate_event => [
-      { :name => :pos,              :length => 8,   :format => "Q"   },
-    ],
-    :query_event => [
-      { :name => :thread_id,        :length => 4,   :format => "V"   },
-      { :name => :elapsed_time,     :length => 4,   :format => "V"   },
-      { :name => :db_length,        :length => 1,   :format => "C"   },
-      { :name => :error_code,       :length => 2,   :format => "v"   },
-    ],
-    :intvar_event => [
-      { :name => :intvar_type,      :length => 1,   :format => "C"   },
-      { :name => :intvar_value,     :length => 8,   :format => "Q"   },
-    ],
-    :xid_event => [
-      { :name => :xid,              :length => 8,   :format => "Q"   },
-    ],
-    :rand_event => [ # Untested
-      { :name => :seed1,            :length => 8,   :format => "Q"   },
-      { :name => :seed2,            :length => 8,   :format => "Q"   },
-    ],
-  }
-
-  class UnsupportedVersionException < Exception; end
-  class MalformedBinlogException < Exception; end
-  class ZeroReadException < Exception; end
-  class ShortReadException < Exception; end
-
-  class Binlog
-    attr_reader :fde
-    attr_accessor :reader
-    attr_accessor :parser
-    attr_accessor :event_field_parser
-    attr_accessor :filter_event_types
-    attr_accessor :filter_flags
-    attr_accessor :max_query_length
-
-    def initialize(reader)
-      @reader = reader
-      @parser = BinlogParser.new(self)
-      @event_field_parser = BinlogEventFieldParser.new(self)
-      @fde = nil
-      @filter_event_types = nil
-      @filter_flags = nil
-      @max_query_length = 1048576
-    end
-
-    # Rewind to the beginning of the log, if supported by the reader. The
-    # reader may throw an exception if rewinding is not supported (e.g. for
-    # a stream-based reader).
-    def rewind
-      reader.rewind
-    end
-
-    # Read fixed fields using format definitions in 'unpack' format provided
-    # in the EVENT_FORMATS hash.
-    def read_fixed_fields(event_type, header)
-      if EVENT_FORMATS.include? event_type
-        parser.read_and_unpack(EVENT_FORMATS[event_type])
-      end
-    end
-
-    # Read dynamic fields or fields that require more processing before being
-    # saved in the event.
-    def read_additional_fields(event_type, header, fields)
-      if event_field_parser.methods.include? event_type.to_s
-        event_field_parser.send(event_type, header, fields)
-      end
-    end
-
-    # Skip the remainder of this event. This can be used to skip an entire
-    # event or merely the parts of the event this library does not understand.
-    def skip_event(header)
-      reader.skip(header)
-    end
-
-    # Read the common header for an event. Every event has a header.
-    def read_event_header
-      header = parser.read_and_unpack(EVENT_HEADER)
-
-      # Merge the read 'flags' bitmap with the EVENT_FLAGS hash to return
-      # the flags by name instead of returning the bitmap as an integer.
-      flags = EVENT_FLAGS.inject([]) do |result, (flag_name, flag_bit_value)|
-        if (header[:flags] & flag_bit_value) != 0
-          result << flag_name
-        end
-        result
-      end
-
-      # Overwrite the integer version of 'flags' with the array of names.
-      header[:flags] = flags
-
-      header
-    end
-
-    # Read the content of the event, which consists of an optional fixed field
-    # portion and an optional dynamic portion.
-    def read_event_content(header)
-      event_type = EVENT_TYPES[header[:event_type]]
-
-      # Read the fixed portion of the event, if it is understood, or there is
-      # one. If not, initialize content with an empty hash instead.
-      content = read_fixed_fields(event_type, header) || {}
-
-      # Read additional fields from the dynamic portion of the event. Some of
-      # these may actually be fixed width but needed more processing in a
-      # function instead of the unpack formats possible in read_fixed_fields.
-      read_additional_fields(event_type, header, content)
-
-      # Anything left unread at this point is skipped based on the event length
-      # provided in the header. In this way, it is possible to skip over events
-      # that are not able to be parsed correctly by this library.
-      skip_event(header)
-
-      content
-    end
-
-    # Scan events until finding one that isn't rejected by the filter rules.
-    # If there are no filter rules, this will return the next event provided
-    # by the reader.
-    def read_event
-      while true
-        skip_this_event = false
-        return nil if reader.end?
-
-        filename = reader.filename
-        position = reader.position
-
-        unless header = read_event_header
-          return nil
-        end
-
-        event_type = EVENT_TYPES[header[:event_type]]
-        
-        if @filter_event_types
-          unless @filter_event_types.include? event_type or
-                  event_type == :format_description_event
-            skip_event(header)
-            skip_this_event = true
-          end
-        end
-        
-        if @filter_flags
-          unless @filter_flags.include? header[:flags]
-            skip_event(header)
-            skip_this_event = true
-          end
-        end
-
-        # Never skip over rotate_event or format_description_event as they
-        # are critical to understanding the format of this event stream.
-        unless [:rotate_event, :format_description_event].include? event_type
-          next if skip_this_event
-        end
-        
-        content = read_event_content(header)
-
-        case event_type
-        when :rotate_event
-          reader.rotate(content[:name], content[:pos])
-        when :format_description_event
-          process_fde(content)
-        end
-
-        break
-      end
-
-      {
-        :type     => event_type,
-        :filename => filename,
-        :position => position,
-        :header   => header,
-        :event    => content,
-      }
-    end
-
-    # Process a format description event, which describes the version of this
-    # file, and the format of events which will appear in this file. This also
-    # provides the version of the MySQL server which generated this file.
-    def process_fde(fde)
-      if (version = fde[:binlog_version]) != 4
-        raise UnsupportedVersionException.new("Binlog version #{version} is not supported")
-      end
-
-      @fde = {
-        :header_length  => fde[:header_length],
-        :binlog_version => fde[:binlog_version],
-        :server_version => fde[:server_version],
-      }
-    end
-
-    # Iterate through all events.
-    def each_event
-      while event = read_event
-        yield event
-      end
-    end
-  end
-end