mysql_binlog 0.1.0

data/bin/mysql_binlog_dump

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require 'pp'
+require 'mysql_binlog'
+
+include MysqlBinlog
+
+b = Binlog.new(BinlogFileReader, ARGV.first)
+
+b.each_event do |event|
+  pp event
+end

data/lib/mysql_binlog/binlog_event_field_parser.rb

@@ -0,0 +1,162 @@
+module MysqlBinlog
+  QUERY_EVENT_STATUS_TYPES = [
+    :flags2,
+    :sql_mode,
+    :catalog,
+    :auto_increment,
+    :charset,
+    :time_zone,
+    :catalog_nz,
+    :lc_time_names,
+    :charset_database,
+    :table_map_for_update,
+  ]
+
+  QUERY_EVENT_FLAGS2 = {
+    :auto_is_null           => 1 << 14,
+    :not_autocommit         => 1 << 19,
+    :no_foreign_key_checks  => 1 << 26,
+    :relaxed_unique_checks  => 1 << 27,
+  }
+
+  class BinlogEventFieldParser
+    attr_accessor :binlog
+    attr_accessor :reader
+    attr_accessor :parser
+
+    def initialize(binlog_instance)
+      @binlog = binlog_instance
+      @reader = binlog_instance.reader
+      @parser = binlog_instance.parser
+      @table_map = {}
+    end
+
+    def rotate_event(header, fields)
+      name_length = reader.remaining(header)
+      fields[:name_length] = name_length
+      fields[:name] = reader.read(name_length)
+    end
+
+    def _query_event_status(header, fields)
+      status = {}
+      end_position = reader.position + fields[:status_length]
+      while reader.position < end_position
+        status_type = QUERY_EVENT_STATUS_TYPES[parser.read_uint8]
+        status[status_type] = case status_type
+        when :flags2
+          flags2 = parser.read_uint32
+          QUERY_EVENT_FLAGS2.inject([]) do |result, (flag_name, flag_bit_value)|
+            if (flags2 & flag_bit_value) != 0
+              result << flag_name
+            end
+            result
+          end
+        when :sql_mode
+          parser.read_uint64
+        when :catalog
+          parser.read_lpstringz
+        when :auto_increment
+          {
+            :increment => parser.read_uint16,
+            :offset    => parser.read_uint16,
+          }
+        when :charset
+          {
+            :character_set_client => parser.read_uint16,
+            :collation_connection => parser.read_uint16,
+            :collation_server     => parser.read_uint16,
+          }
+        when :time_zone
+          parser.read_lpstring
+        when :catalog_nz
+          parser.read_lpstring
+        when :lc_time_names
+          parser.read_uint16
+        when :charset_database
+          parser.read_uint16
+        when :table_map_for_update
+          parser.read_uint64
+        end
+      end
+      status
+    end
+
+    def query_event(header, fields)
+      fields[:status] = _query_event_status(header, fields)
+      fields.delete :status_length
+      fields[:db] = parser.read_nstringz(fields[:db_length])
+      fields.delete :db_length
+      query_length = reader.remaining(header)
+      fields[:query] = reader.read([query_length, binlog.max_query_length].min)
+    end
+
+    def intvar_event(header, fields)
+      case fields[:intvar_type]
+      when 1
+        fields[:intvar_name] = :last_insert_id
+      when 2
+        fields[:intvar_name] = :insert_id
+      else
+        fields[:intvar_name] = nil
+      end
+    end
+
+    def table_map_event(header, fields)
+      fields[:table_id] = parser.read_uint48
+      fields[:flags] = parser.read_uint16
+      map_entry = @table_map[fields[:table_id]] = {}
+      map_entry[:db] = parser.read_lpstringz
+      map_entry[:table] = parser.read_lpstringz
+      columns = parser.read_varint
+      columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] }
+      columns_metadata = parser.read_lpstring
+      columns_nullable = parser.read_bit_array(columns)
+
+      map_entry[:columns] = columns.times.map do |c|
+        {
+          :type     => columns_type[c],
+          :nullable => columns_nullable[c],
+        }
+      end
+
+      fields[:map_entry] = map_entry
+    end
+
+    def _generic_rows_event_row_images(header, fields)
+      row_images = []
+      end_position = reader.position + reader.remaining(header)
+      while reader.position < end_position
+        row_image = []
+        columns_null = parser.read_bit_array(fields[:table][:columns].size)
+        fields[:table][:columns].each_with_index do |column, column_index|
+          if !fields[:columns_used][column_index]
+            row_image << nil
+          elsif columns_null[column_index]
+            row_image << { column => nil }
+          else
+            row_image << { column => parser.read_mysql_type(column[:type]) }
+          end
+        end
+        row_images << row_image
+      end
+      row_images
+    end
+
+    def generic_rows_event(header, fields)
+      table_id = parser.read_uint48
+      fields[:table] = @table_map[table_id]
+      fields[:flags] = parser.read_uint16
+      columns = parser.read_varint
+      fields[:columns_used] = parser.read_bit_array(columns)
+      if EVENT_TYPES[header[:event_type]] == :update_rows_event
+        fields[:columns_update] = parser.read_bit_array(columns)
+      end
+      fields[:row_image] = _generic_rows_event_row_images(header, fields)
+      fields.delete :columns_used
+    end
+    alias :write_rows_event  :generic_rows_event
+    alias :update_rows_event :generic_rows_event
+    alias :delete_rows_event :generic_rows_event
+
+  end
+end

data/lib/mysql_binlog/binlog_parser.rb

@@ -0,0 +1,179 @@
+module MysqlBinlog
+  MYSQL_TYPES_HASH = {
+    :decimal         => 0,
+    :tiny            => 1,
+    :short           => 2,
+    :long            => 3,
+    :float           => 4,
+    :double          => 5,
+    :null            => 6,
+    :timestamp       => 7,
+    :longlong        => 8,
+    :int24           => 9,
+    :date            => 10,
+    :time            => 11,
+    :datetime        => 12,
+    :year            => 13,
+    :newdate         => 14,
+    :varchar         => 15,
+    :bit             => 16,
+    :newdecimal      => 246,
+    :enum            => 247,
+    :set             => 248,
+    :tiny_blob       => 249,
+    :medium_blob     => 250,
+    :long_blob       => 251,
+    :blob            => 252,
+    :var_string      => 253,
+    :string          => 254,
+    :geometry        => 255,
+  }
+  
+  MYSQL_TYPES = MYSQL_TYPES_HASH.inject(Array.new(256)) do |type_array, item|
+   type_array[item[1]] = item[0]
+   type_array
+  end
+
+  class BinlogParser
+    attr_accessor :binlog
+    attr_accessor :reader
+
+    def initialize(binlog_instance)
+      @format_cache = {}
+      @binlog = binlog_instance
+      @reader = binlog_instance.reader
+    end
+
+    def read_uint8
+      reader.read(1).unpack("C").first
+    end
+
+    def read_uint16
+      reader.read(2).unpack("v").first
+    end
+
+    def read_uint24
+      a, b, c = reader.read(3).unpack("CCC")
+      a + (b << 8) + (c << 16)
+    end
+
+    def read_uint32
+      reader.read(4).unpack("V").first
+    end
+
+    def read_uint48
+      a, b, c = reader.read(6).unpack("vvv")
+      a + (b << 16) + (c << 32)
+    end
+
+    def read_uint64
+      reader.read(8).unpack("Q").first
+    end
+
+    def read_float
+      reader.read(4).unpack("g").first
+    end
+
+    def read_double
+      reader.read(8).unpack("G").first
+    end
+
+    def read_varint
+      # Cheating for now.
+      read_uint8
+    end
+
+    def read_lpstring
+      length = reader.read(1).unpack("C").first
+      reader.read(length)
+    end
+
+    def read_lpstringz
+      string = read_lpstring
+      reader.read(1) # null
+      string
+    end
+
+    def read_nstring(length)
+      string = reader.read(length)
+      string
+    end
+
+    def read_nstringz(length)
+      string = reader.read(length)
+      reader.read(1) # null
+      string
+    end
+
+    def read_uint8_array(length)
+      reader.read(length).bytes.to_a
+    end
+
+    def read_bit_array(length)
+      data = reader.read((length+7)/8)
+      data.unpack("b*").first.bytes.to_a.map { |i| (i-48) == 1 }.shift(length)
+    end
+
+    def read_and_unpack(format_description)
+      @format_cache[format_description] ||= {}
+      this_format = @format_cache[format_description][:format] ||= 
+        format_description.inject("") { |o, f| o+(f[:format] || "") }
+      this_length = @format_cache[format_description][:length] ||=
+        format_description.inject(0)  { |o, f| o+(f[:length] || 0) }
+
+      fields = {}
+
+      fields_array = reader.read(this_length).unpack(this_format)
+      format_description.each_with_index do |field, index| 
+        fields[field[:name]] = fields_array[index]
+      end
+
+      fields
+    end
+
+    def read_mysql_type(column_type)
+      case column_type
+      #when :decimal
+      when :tiny
+        read_uint8
+      when :short
+        read_uint16
+      when :int24
+        read_uint24
+      when :long
+        read_uint32
+      when :longlong
+        read_uint64
+      when :string
+        length = read_varint
+        read_nstring(length)
+
+      when :float
+        read_float
+      when :double
+        read_double
+      #when :null
+      when :timestamp
+        read_uint32
+      #when :date
+      #when :time
+      #when :datetime
+      #when :year
+      #when :newdate
+      #when :varchar
+      #when :bit
+      #when :newdecimal
+      #when :enum
+      #when :set
+      #when :tiny_blob
+      #when :medium_blob
+      #when :long_blob
+      #when :blob
+      #when :var_string
+      #when :string
+      #when :geometry
+      end
+    end
+
+  end
+end

data/lib/mysql_binlog/mysql_binlog.rb

@@ -0,0 +1,234 @@
+require 'mysql_binlog/binlog_parser'
+
+module MysqlBinlog
+  def puts_hex(data)
+    hex = data.bytes.each_slice(24).inject("") do |string, slice|
+      string << slice.map { |b| "%02x" % b }.join(" ") + "\n"
+      string
+    end
+    puts hex
+  end
+
+  MAGIC = [
+    { :name => :magic,            :length => 4,   :format => "V"   },
+  ]
+
+  EVENT_HEADER = [
+    { :name => :timestamp,        :length => 4,   :format => "V"   },
+    { :name => :event_type,       :length => 1,   :format => "C"   },
+    { :name => :server_id,        :length => 4,   :format => "V"   },
+    { :name => :event_length,     :length => 4,   :format => "V"   },
+    { :name => :next_position,    :length => 4,   :format => "V"   },
+    { :name => :flags,            :length => 2,   :format => "v"   },
+  ]
+
+  EVENT_TYPES = [
+    :unknown_event,             #  0
+    :start_event_v3,            #  1
+    :query_event,               #  2
+    :stop_event,                #  3
+    :rotate_event,              #  4
+    :intvar_event,              #  5
+    :load_event,                #  6
+    :slave_event,               #  7
+    :create_file_event,         #  8
+    :append_block_event,        #  9
+    :exec_load_event,           # 10
+    :delete_file_event,         # 11
+    :new_load_event,            # 12
+    :rand_event,                # 13
+    :user_var_event,            # 14
+    :format_description_event,  # 15
+    :xid_event,                 # 16
+    :begin_load_query_event,    # 17
+    :execute_load_query_event,  # 18
+    :table_map_event,           # 19
+    :pre_ga_write_rows_event,   # 20
+    :pre_ga_update_rows_event,  # 21
+    :pre_ga_delete_rows_event,  # 22
+    :write_rows_event,          # 23
+    :update_rows_event,         # 24
+    :delete_rows_event,         # 25
+    :incident_event,            # 26
+    :heartbeat_log_event,       # 27
+  ]
+
+  EVENT_FLAGS = {
+    :binlog_in_use   => 0x01,
+    :thread_specific => 0x04,
+    :suppress_use    => 0x08,
+    :artificial      => 0x20,
+    :relay_log       => 0x40,
+  }
+
+  EVENT_FORMATS = {
+    :format_description_event => [
+      { :name => :binlog_version,   :length => 2,   :format => "v"   },
+      { :name => :server_version,   :length => 50,  :format => "A50" },
+      { :name => :create_timestamp, :length => 4,   :format => "V"   },
+      { :name => :header_length,    :length => 1,   :format => "C"   },
+    ],
+    :rotate_event => [
+      { :name => :pos,              :length => 8,   :format => "Q"   },
+    ],
+    :query_event => [
+      { :name => :thread_id,        :length => 4,   :format => "V"   },
+      { :name => :elapsed_time,     :length => 4,   :format => "V"   },
+      { :name => :db_length,        :length => 1,   :format => "C"   },
+      { :name => :error_code,       :length => 2,   :format => "v"   },
+      { :name => :status_length,    :length => 2,   :format => "v"   },
+    ],
+    :intvar_event => [
+      { :name => :intvar_type,      :length => 1,   :format => "C"   },
+      { :name => :intvar_value,     :length => 8,   :format => "Q"   },
+    ],
+    :xid_event => [
+      { :name => :xid,              :length => 8,   :format => "Q"   },
+    ],
+    :rand_event => [ # Untested
+      { :name => :seed1,            :length => 8,   :format => "Q"   },
+      { :name => :seed2,            :length => 8,   :format => "Q"   },
+    ],
+  }
+
+  class UnsupportedVersionException < Exception; end
+  class MalformedBinlogException < Exception; end
+  class ZeroReadException < Exception; end
+  class ShortReadException < Exception; end
+
+  class Binlog
+    attr_reader :fde
+    attr_accessor :reader
+    attr_accessor :parser
+    attr_accessor :event_field_parser
+    attr_accessor :filter_event_types
+    attr_accessor :filter_flags
+    attr_accessor :max_query_length
+
+    def initialize(reader_class, *args)
+      @reader = reader_class.new(*args)
+      @parser = BinlogParser.new(self)
+      @event_field_parser = BinlogEventFieldParser.new(self)
+      @fde = nil
+      @filter_event_types = nil
+      @filter_flags = nil
+      @max_query_length = 1048576
+    end
+
+    def rewind
+      reader.rewind
+      read_file_header
+    end
+
+    def read_additional_fields(event_type, header, fields)
+      if event_field_parser.methods.include? event_type.to_s
+        event_field_parser.send(event_type, header, fields)
+      end
+    end
+
+    def skip_event(header)
+      reader.skip(header)
+    end
+
+    def read_event_header
+      header = parser.read_and_unpack(EVENT_HEADER)
+      flags = EVENT_FLAGS.inject([]) do |result, (flag_name, flag_bit_value)|
+        if (header[:flags] & flag_bit_value) != 0
+          result << flag_name
+        end
+        result
+      end
+      header[:flags] = flags
+      header
+    end
+
+    def read_event_content(header)
+      content = nil
+
+      event_type = EVENT_TYPES[header[:event_type]]
+      if EVENT_FORMATS.include? event_type
+        content = parser.read_and_unpack(EVENT_FORMATS[event_type])
+      else
+        content = {}
+      end
+
+      read_additional_fields(event_type, header, content)
+
+      skip_event(header)
+      content
+    end
+
+    def read_event
+      while true
+        skip_this_event = false
+        return nil if reader.end?
+
+        filename = reader.filename
+        position = reader.position
+
+        unless header = read_event_header
+          return nil
+        end
+
+        event_type = EVENT_TYPES[header[:event_type]]
+        
+        if @filter_event_types
+          unless @filter_event_types.include? event_type or
+                  event_type == :format_description_event
+            skip_event(header)
+            skip_this_event = true
+          end
+        end
+        
+        if @filter_flags
+          unless @filter_flags.include? header[:flags]
+            skip_event(header)
+            skip_this_event = true
+          end
+        end
+
+        unless [:rotate_event, :format_description_event].include? event_type
+          next if skip_this_event
+        end
+        
+        content = read_event_content(header)
+        content
+
+        case event_type
+        when :rotate_event
+          reader.rotate(content[:name], content[:pos])
+        when :format_description_event
+          process_fde(content)
+        end
+
+        break
+      end
+
+      {
+        :type     => event_type,
+        :filename => filename,
+        :position => position,
+        :header   => header,
+        :event    => content,
+      }
+    end
+
+    def process_fde(fde)
+      if (version = fde[:binlog_version]) != 4
+        raise UnsupportedVersionException.new("Binlog version #{version} is not supported")
+      end
+
+      @fde = {
+        :header_length  => fde[:header_length],
+        :binlog_version => fde[:binlog_version],
+        :server_version => fde[:server_version],
+      }
+    end
+  
+    def each_event
+      while event = read_event
+        yield event
+      end
+    end
+  end
+end

data/lib/mysql_binlog/reader/binlog_file_reader.rb

@@ -0,0 +1,65 @@
+module MysqlBinlog
+  class BinlogFileReader
+    def initialize(filename)
+      @filename = filename
+      @binlog = nil
+      
+      open_file(filename)
+    end
+
+    def open_file(filename)
+      @filename = filename
+      @binlog = File.open(filename, mode="r")
+
+      if (magic = read(4).unpack("V").first) != 1852400382
+        raise MalformedBinlogException.new("Magic number #{magic} is incorrect")
+      end
+    end
+
+    def rotate(filename, position)
+      open_file(filename)
+      seek(position)
+    end
+
+    def filename
+      @filename
+    end
+
+    def position
+      @binlog.tell
+    end
+
+    def rewind
+      @binlog.rewind
+    end
+
+    def seek(pos)
+      @binlog.seek(pos)
+    end
+  
+    def end?
+      @binlog.eof?
+    end
+
+    def remaining(header)
+      header[:next_position] - @binlog.tell
+    end
+
+    def skip(header)
+      seek(header[:next_position])
+    end
+
+    def read(length)
+      return "" if length == 0
+      data = @binlog.read(length)
+      if !data
+        raise MalformedBinlogException.new
+      elsif data.length == 0
+        raise ZeroReadException.new
+      elsif data.length < length
+        raise ShortReadException.new
+      end
+      data
+    end
+  end
+end

data/lib/mysql_binlog/reader/binlog_stream_reader.rb

@@ -0,0 +1,64 @@
+module MysqlBinlog
+  class BinlogStreamReader
+    def initialize(connection, filename, position)
+      require 'mysql_binlog_dump'
+      @filename = nil
+      @position = nil
+      @packet_data = nil
+      @packet_pos  = nil
+      @connection = connection
+      MysqlBinlogDump.binlog_dump(connection, filename, position)
+    end
+
+    def rotate(filename, position)
+      puts "rotate called with #{filename}:#{position}"
+      @filename = filename
+      @position = position
+    end
+
+    def filename
+      @filename
+    end
+    
+    def position
+      @position
+    end
+
+    def rewind
+      false
+    end
+
+    def tell
+      @packet_pos
+    end
+
+    def end?
+      false
+    end
+
+    def remaining(header)
+      @packet_data.length - @packet_pos
+    end
+
+    def skip(header)
+      @packet_data = nil
+      @packet_pos  = nil
+    end
+
+    def read_packet
+      @packet_data = MysqlBinlogDump.next_packet(@connection)
+      @packet_pos  = 0
+    end
+
+    def read(length)
+      unless @packet_data
+        read_packet
+        return nil unless @packet_data
+      end
+      pos = @packet_pos
+      @position   += length if @position
+      @packet_pos += length
+      @packet_data[pos...(pos+length)]
+    end
+  end
+end

data/lib/mysql_binlog.rb

@@ -0,0 +1,4 @@
+require 'mysql_binlog/mysql_binlog'
+require 'mysql_binlog/binlog_event_field_parser'
+require 'mysql_binlog/reader/binlog_file_reader'
+require 'mysql_binlog/reader/binlog_stream_reader'

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification 
+name: mysql_binlog
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Jeremy Cole
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-06-24 00:00:00 Z
+dependencies: []
+
+description: Library for parsing MySQL binary logs in Ruby
+email: jeremy@jcole.us
+executables: 
+- mysql_binlog_dump
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/mysql_binlog.rb
+- lib/mysql_binlog/mysql_binlog.rb
+- lib/mysql_binlog/binlog_parser.rb
+- lib/mysql_binlog/binlog_event_field_parser.rb
+- lib/mysql_binlog/reader/binlog_file_reader.rb
+- lib/mysql_binlog/reader/binlog_stream_reader.rb
+- bin/mysql_binlog_dump
+homepage: http://jcole.us/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: MySQL Binary Log Parser
+test_files: []
+