mysql_binlog 0.3.2 → 0.3.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d4cb94c1699fea0fcfb90ef92d1d09d15dadb5a5
+  data.tar.gz: d6dcf06a12be0f42347344d250156f34a2ecf962
+SHA512:
+  metadata.gz: 7920efabe0e87c67d041d6712afeb2d27cd72978504e314ab4b9ee78fdeeaff2895e363c896a45972d3c0eeabd6ad84495c8f591013e137ed65893762111291b
+  data.tar.gz: c82e4f6b2fbf47de40505c70a310eca59e96f03c2cd85da4435cb69d582d80a3a597ea865ef43257547a927df83b03caa42b29bcc29f36b374b0e7a0f649b12c

data/bin/mysql_binlog_dump

@@ -15,13 +15,16 @@ def usage(exit_code, message = nil)
 
 Usage:
   To read from a binary log file on disk:
-    mysql_binlog_dump [options] -f <filename>
+    mysql_binlog_dump [options] <filename(s)>
 
   --help, -?
     Show this help.
 
   --file, -f <filename>
-    Read from a binary log file on disk.
+    Read from a binary log file on disk (deprecated).
+
+  --checksum, -c
+    Enable CRC32 checksums.
 
   --debug, -d
     Debug reading from the binary log, showing calls into the reader and the
@@ -45,13 +48,16 @@ end
 
 @options = OpenStruct.new
 @options.file = nil
+@options.checksum = nil
 @options.debug = false
 @options.tail = false
 @options.rotate = false
+@options.filenames = []
 
 getopt_options = [
   [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
   [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--checksum",         "-c",     GetoptLong::NO_ARGUMENT ],
   [ "--debug",            "-d",     GetoptLong::NO_ARGUMENT ],
   [ "--tail",             "-t",     GetoptLong::NO_ARGUMENT ],
   [ "--rotate",           "-r",     GetoptLong::NO_ARGUMENT ],
@@ -64,7 +70,9 @@ getopt.each do |opt, arg|
   when "--help"
     usage 0
   when "--file"
-    @options.file = arg
+    @options.filenames << arg
+  when "--checksum"
+    @options.checksum = :crc32
   when "--debug"
     @options.debug = true
   when "--tail"
@@ -74,34 +82,25 @@ getopt.each do |opt, arg|
   end
 end
 
-unless @options.file
-  usage 1, "A file must be provided with --file/-f"
-end
+@options.filenames.concat(ARGV)
 
-reader = BinlogFileReader.new(@options.file)
-if @options.debug
-  reader = DebuggingReader.new(reader, :data => true, :calls => true)
+if @options.filenames.empty?
+  usage 1, "One or more filenames must be provided"
 end
-binlog = Binlog.new(reader)
 
-if @options.tail
-  reader.tail = true
-else
-  reader.tail = false
-end
+@options.filenames.each do |filename|
+  reader = BinlogFileReader.new(filename)
+  if @options.debug
+    reader = DebuggingReader.new(reader, :data => true, :calls => true)
+  end
+  binlog = Binlog.new(reader)
+  binlog.checksum = @options.checksum
 
-if @options.rotate
-  binlog.ignore_rotate = false
-else
-  binlog.ignore_rotate = true
-end
+  reader.tail = @options.tail
+  binlog.ignore_rotate = !@options.rotate
 
-binlog.each_event do |event|
-  puts "%-30s%-20s%20d" % [
-    event[:type],
-    event[:filename],
-    event[:position],
-  ]
-  #pp event
-  #puts
-end
+  binlog.each_event do |event|
+    pp event
+    puts
+  end
+end

data/lib/mysql_binlog/binlog.rb

@@ -51,6 +51,7 @@ module MysqlBinlog
     attr_accessor :filter_flags
     attr_accessor :ignore_rotate
     attr_accessor :max_query_length
+    attr_accessor :checksum
 
     def initialize(reader)
       @reader = reader
@@ -61,6 +62,7 @@ module MysqlBinlog
       @filter_flags = nil
       @ignore_rotate = false
       @max_query_length = 1048576
+      @checksum = :nil
     end
 
     # Rewind to the beginning of the log, if supported by the reader. The
@@ -81,10 +83,16 @@ module MysqlBinlog
     def read_event_fields(header)
       # Delegate the parsing of the event content to a method of the same name
       # in BinlogEventParser.
-      if event_parser.methods.include? header[:event_type].to_s
+      if event_parser.methods.map(&:to_sym).include? header[:event_type]
         fields = event_parser.send(header[:event_type], header)
       end
 
+      unless fields
+        fields = {
+          payload: reader.read(header[:payload_length]),
+        }
+      end
+
       # Check if we've read past the end of the event. This is normally because
       # of an unsupported substructure in the event causing field misalignment
       # or a bug in the event reader method in BinlogEventParser. This may also
@@ -103,6 +111,19 @@ module MysqlBinlog
     end
     private :read_event_fields
 
+    def checksum_length
+      case @checksum
+      when :crc32
+        4
+      else
+        0
+      end
+    end
+
+    def payload_length(header)
+      @fde ? (header[:event_length] - @fde[:header_length] - checksum_length) : 0
+    end
+
     # Scan events until finding one that isn't rejected by the filter rules.
     # If there are no filter rules, this will return the next event provided
     # by the reader.
@@ -119,6 +140,18 @@ module MysqlBinlog
           return nil
         end
 
+        # Skip the remaining part of the header which might not have been
+        # parsed.
+        if @fde
+          reader.seek(position + @fde[:header_length])
+          header[:payload_length] = payload_length(header)
+          header[:payload_end] = position + @fde[:header_length] + payload_length(header)
+        else
+          header[:payload_length] = 0
+          header[:payload_end] = header[:next_position]
+        end
+
+
         if @filter_event_types
           unless @filter_event_types.include? header[:event_type]
             skip_this_event = true

data/lib/mysql_binlog/binlog_event_parser.rb

@@ -26,11 +26,23 @@ module MysqlBinlog
     :pre_ga_write_rows_event    => 20,  # (deprecated)
     :pre_ga_update_rows_event   => 21,  # (deprecated)
     :pre_ga_delete_rows_event   => 22,  # (deprecated)
-    :write_rows_event           => 23,  #
-    :update_rows_event          => 24,  #
-    :delete_rows_event          => 25,  #
+    :write_rows_event_v1        => 23,  #
+    :update_rows_event_v1       => 24,  #
+    :delete_rows_event_v1       => 25,  #
     :incident_event             => 26,  #
     :heartbeat_log_event        => 27,  #
+    :ignorable_log_event        => 28,
+    :rows_query_log_event       => 29,
+    :write_rows_event_v2        => 30,
+    :update_rows_event_v2       => 31,
+    :delete_rows_event_v2       => 32,
+    :gtid_log_event             => 33,
+    :anonymous_gtid_log_event   => 34,
+    :previous_gtids_log_event   => 35,
+    :transaction_context_event  => 36,
+    :view_change_event          => 37,
+    :xa_prepare_log_event       => 38,
+
     :table_metadata_event       => 50,  # Only in Twitter MySQL
   }
 
@@ -44,9 +56,9 @@ module MysqlBinlog
   # have an identical structure, this list can be used by other programs to
   # know which events can be treated as row events.
   ROW_EVENT_TYPES = [
-    :write_rows_event,
-    :update_rows_event,
-    :delete_rows_event,
+    :write_rows_event_v1,
+    :update_rows_event_v1,
+    :delete_rows_event_v1,
   ]
 
   # Values for the +flags+ field that may appear in binary logs. There are
@@ -55,11 +67,14 @@ module MysqlBinlog
   #
   # Defined in sql/log_event.h line ~448
   EVENT_HEADER_FLAGS = {
-    :binlog_in_use   => 0x01, # LOG_EVENT_BINLOG_IN_USE_F
-    :thread_specific => 0x04, # LOG_EVENT_THREAD_SPECIFIC_F
-    :suppress_use    => 0x08, # LOG_EVENT_SUPPRESS_USE_F
-    :artificial      => 0x20, # LOG_EVENT_ARTIFICIAL_F
-    :relay_log       => 0x40, # LOG_EVENT_RELAY_LOG_F
+    :binlog_in_use    => 0x0001, # LOG_EVENT_BINLOG_IN_USE_F
+    :thread_specific  => 0x0004, # LOG_EVENT_THREAD_SPECIFIC_F
+    :suppress_use     => 0x0008, # LOG_EVENT_SUPPRESS_USE_F
+    :artificial       => 0x0020, # LOG_EVENT_ARTIFICIAL_F
+    :relay_log        => 0x0040, # LOG_EVENT_RELAY_LOG_F
+    :ignorable        => 0x0080, # LOG_EVENT_IGNORABLE_F
+    :no_filter        => 0x0100, # LOG_EVENT_NO_FILTER_F
+    :mts_isolate      => 0x0200, # LOG_EVENT_MTS_ISOLATE_F
   }
 
   # A mapping array for all values that may appear in the +status+ field of
@@ -79,8 +94,15 @@ module MysqlBinlog
     :table_map_for_update,      #  9 (Q_TABLE_MAP_FOR_UPDATE_CODE)
     :master_data_written,       # 10 (Q_MASTER_DATA_WRITTEN_CODE)
     :invoker,                   # 11 (Q_INVOKER)
+    :updated_db_names,          # 12 (Q_UPDATED_DB_NAMES)
+    :microseconds,              # 13 (Q_MICROSECONDS)
+    :commit_ts,                 # 14 (Q_COMMIT_TS)
+    :commit_ts2,                # 15
+    :explicit_defaults_for_timestamp, # 16
   ]
 
+  QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS = 254
+
   # A mapping hash for all values that may appear in the +flags2+ field of
   # a query_event.
   #
@@ -181,11 +203,13 @@ module MysqlBinlog
     def event_header
       header = {}
       header[:timestamp]      = parser.read_uint32
-      header[:event_type]     = EVENT_TYPES[parser.read_uint8]
+      event_type = parser.read_uint8
+      header[:event_type]     = EVENT_TYPES[event_type] || "unknown_#{event_type}".to_sym
       header[:server_id]      = parser.read_uint32
       header[:event_length]   = parser.read_uint32
       header[:next_position]  = parser.read_uint32
       header[:flags] = parser.read_uint_bitmap_by_size_and_name(2, EVENT_HEADER_FLAGS)
+
       header
     end
 
@@ -212,6 +236,25 @@ module MysqlBinlog
       fields
     end
 
+    def _query_event_status_updated_db_names
+      db_count = parser.read_uint8
+      return nil if db_count == QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS
+
+      db_names = []
+      db_count.times do |n|
+        db_name = ""
+        loop do
+          c = reader.read(1)
+          break if c == "\0"
+          db_name << c
+        end
+        db_names << db_name
+      end
+
+      db_names
+    end
+    private :_query_event_status_updated_db_names
+
     # Parse a dynamic +status+ structure within a query_event, which consists
     # of a status_length (uint16) followed by a number of status variables
     # (determined by the +status_length+) each of which consist of:
@@ -223,7 +266,8 @@ module MysqlBinlog
       status_length = parser.read_uint16
       end_position = reader.position + status_length
       while reader.position < end_position
-        status_type = QUERY_EVENT_STATUS_TYPES[parser.read_uint8]
+        status_type_id = parser.read_uint8
+        status_type = QUERY_EVENT_STATUS_TYPES[status_type_id]
         status[status_type] = case status_type
         when :flags2
           parser.read_uint_bitmap_by_size_and_name(4, QUERY_EVENT_FLAGS2)
@@ -252,6 +296,12 @@ module MysqlBinlog
           parser.read_uint16
         when :table_map_for_update
           parser.read_uint64
+        when :updated_db_names
+          _query_event_status_updated_db_names
+        when :commit_ts
+          parser.read_uint64
+        else
+          raise "Unknown status type #{status_type_id}"
         end
       end
 
@@ -334,7 +384,7 @@ module MysqlBinlog
           :precision => parser.read_uint8,
           :decimals  => parser.read_uint8,
         }
-      when :blob, :geometry
+      when :blob, :geometry, :json
         { :length_size => parser.read_uint8 }
       when :string, :var_string
         # The :string type sets a :real_type field to indicate the actual type
@@ -350,6 +400,10 @@ module MysqlBinlog
         else
           { :max_length  => (((metadata >> 4) & 0x300) ^ 0x300) + (metadata & 0x00ff) }
         end
+      when :timestamp2, :datetime2, :time2
+        {
+          :decimals => parser.read_uint8,
+        }
       end
     end
     private :_table_map_event_column_metadata_read
@@ -376,7 +430,7 @@ module MysqlBinlog
       map_entry[:db] = parser.read_lpstringz
       map_entry[:table] = parser.read_lpstringz
       columns = parser.read_varint
-      columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] }
+      columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] || "unknown_#{c}".to_sym }
       columns_metadata = _table_map_event_column_metadata(columns_type)
       columns_nullable = parser.read_bit_array(columns)
 
@@ -415,8 +469,9 @@ module MysqlBinlog
       fields[:flags] = parser.read_uint16
       fields[:columns] = columns.times.map do |c|
         descriptor_length = parser.read_uint32
+        column_type = parser.read_uint8
         @table_map[table_id][:columns][c][:description] = {
-          :type => MYSQL_TYPES[parser.read_uint8],
+          :type => MYSQL_TYPES[column_type] || "unknown_#{column_type}".to_sym,
           :length => parser.read_uint32,
           :scale => parser.read_uint8,
           :character_set => COLLATION[parser.read_uint16],
@@ -433,41 +488,62 @@ module MysqlBinlog
     # Parse a single row image, which is comprised of a series of columns. Not
     # all columns are present in the row image, the columns_used array of true
     # and false values identifies which columns are present.
-    def _generic_rows_event_row_image(header, fields, columns_used)
+    def _generic_rows_event_row_image_v1(header, fields, columns_used)
       row_image = []
+      start_position = reader.position
       columns_null = parser.read_bit_array(fields[:table][:columns].size)
       fields[:table][:columns].each_with_index do |column, column_index|
+        #puts "column #{column_index} #{column}: used=#{columns_used[column_index]}, null=#{columns_null[column_index]}"
         if !columns_used[column_index]
           row_image << nil
         elsif columns_null[column_index]
           row_image << { column_index => nil }
         else
+          value = parser.read_mysql_type(column[:type], column[:metadata])
           row_image << {
-            column_index =>
-              parser.read_mysql_type(column[:type], column[:metadata])
+            column_index => value,
           }
         end
       end
-      row_image
+      end_position = reader.position
+
+      {
+        image: row_image,
+        size: end_position-start_position
+      }
+    end
+    private :_generic_rows_event_row_image_v1
+
+    def diff_row_images(before, after)
+      diff = {}
+      before.each_with_index do |before_column, index|
+        after_column = after[index]
+        before_value = before_column.first[1]
+        after_value = after_column.first[1]
+        if before_value != after_value
+          diff[index] = { before: before_value, after: after_value }
+        end
+      end
+      diff
     end
-    private :_generic_rows_event_row_image
 
     # Parse the row images present in a row-based replication row event. This
     # is rather incomplete right now due missing support for many MySQL types,
     # but can parse some basic events.
-    def _generic_rows_event_row_images(header, fields, columns_used)
+    def _generic_rows_event_row_images_v1(header, fields, columns_used)
       row_images = []
       end_position = reader.position + reader.remaining(header)
       while reader.position < end_position
         row_image = {}
         case header[:event_type]
-        when :write_rows_event
-          row_image[:after]  = _generic_rows_event_row_image(header, fields, columns_used[:after])
-        when :delete_rows_event
-          row_image[:before] = _generic_rows_event_row_image(header, fields, columns_used[:before])
-        when :update_rows_event
-          row_image[:before] = _generic_rows_event_row_image(header, fields, columns_used[:before])
-          row_image[:after]  = _generic_rows_event_row_image(header, fields, columns_used[:after])
+        when :write_rows_event_v1
+          row_image[:after]  = _generic_rows_event_row_image_v1(header, fields, columns_used[:after])
+        when :delete_rows_event_v1
+          row_image[:before] = _generic_rows_event_row_image_v1(header, fields, columns_used[:before])
+        when :update_rows_event_v1
+          row_image[:before] = _generic_rows_event_row_image_v1(header, fields, columns_used[:before])
+          row_image[:after]  = _generic_rows_event_row_image_v1(header, fields, columns_used[:after])
+          row_image[:diff] = diff_row_images(row_image[:before][:image], row_image[:after][:image])
         end
         row_images << row_image
       end
@@ -481,7 +557,7 @@ module MysqlBinlog
 
       row_images
     end
-    private :_generic_rows_event_row_images
+    private :_generic_rows_event_row_images_v1
 
     # Parse fields for any of the row-based replication row events:
     # * +Write_rows+ which is used for +INSERT+.
@@ -491,7 +567,7 @@ module MysqlBinlog
     # Implemented in sql/log_event.cc line ~8039
     # in Rows_log_event::write_data_header
     # and Rows_log_event::write_data_body
-    def generic_rows_event(header)
+    def generic_rows_event_v1(header)
       fields = {}
       table_id = parser.read_uint48
       fields[:table] = @table_map[table_id]
@@ -499,21 +575,78 @@ module MysqlBinlog
       columns = parser.read_varint
       columns_used = {}
       case header[:event_type]
-      when :write_rows_event
+      when :write_rows_event_v1
         columns_used[:after]  = parser.read_bit_array(columns)
-      when :delete_rows_event
+      when :delete_rows_event_v1
         columns_used[:before] = parser.read_bit_array(columns)
-      when :update_rows_event
+      when :update_rows_event_v1
         columns_used[:before] = parser.read_bit_array(columns)
         columns_used[:after]  = parser.read_bit_array(columns)
       end
-      fields[:row_image] = _generic_rows_event_row_images(header, fields, columns_used)
+      fields[:row_image] = _generic_rows_event_row_images_v1(header, fields, columns_used)
       fields
     end
 
-    alias :write_rows_event  :generic_rows_event
-    alias :update_rows_event :generic_rows_event
-    alias :delete_rows_event :generic_rows_event
+    alias :write_rows_event_v1  :generic_rows_event_v1
+    alias :update_rows_event_v1 :generic_rows_event_v1
+    alias :delete_rows_event_v1 :generic_rows_event_v1
+
+    def rows_query_log_event(header)
+      reader.read(1) # skip useless byte length which is unused
+      { query: reader.read(header[:payload_length]-1) }
+    end
+
+    def in_hex(bytes)
+      bytes.each_byte.map { |c| "%02x" % c.ord }.join
+    end
+
+    def format_gtid_sid(sid)
+      [0..3, 4..5, 6..7, 8..9, 10..15].map { |r| in_hex(sid[r]) }.join("-")
+    end
 
+    # 6d9190a2-cca6-11e8-aa8c-42010aef0019:551845019
+    def format_gtid(sid, gno_or_ivs)
+      "#{format_gtid_sid(sid)}:#{gno_or_ivs}"
+    end
+
+    def previous_gtids_log_event(header)
+      n_sids = parser.read_uint64
+
+      gtids = []
+      n_sids.times do
+        sid = parser.read_nstring(16)
+        n_ivs = parser.read_uint64
+        ivs = []
+        n_ivs.times do
+          iv_start = parser.read_uint64
+          iv_end = parser.read_uint64
+          ivs << "#{iv_start}-#{iv_end}"
+        end
+        gtids << format_gtid(sid, ivs.join(":"))
+      end
+
+      {
+        previous_gtids: gtids
+      }
+    end
+
+    def gtid_log_event(header)
+      flags = parser.read_uint8
+      sid = parser.read_nstring(16)
+      gno = parser.read_uint64
+      lts_type = parser.read_uint8
+      lts_last_committed = parser.read_uint64
+      lts_sequence_number = parser.read_uint64
+
+      {
+        flags: flags,
+        gtid: format_gtid(sid, gno),
+        lts: {
+          type: lts_type,
+          last_committed: lts_last_committed,
+          sequence_number: lts_sequence_number,
+        },
+      }
+    end
   end
 end

data/lib/mysql_binlog/binlog_field_parser.rb

@@ -1,3 +1,5 @@
+require 'bigdecimal'
+
 module MysqlBinlog
   # All MySQL types mapping to their integer values.
   MYSQL_TYPES_HASH = {
@@ -18,6 +20,10 @@ module MysqlBinlog
     :newdate         => 14,
     :varchar         => 15,
     :bit             => 16,
+    :timestamp2      => 17,
+    :datetime2       => 18,
+    :time2           => 19,
+    :json            => 245,
     :newdecimal      => 246,
     :enum            => 247,
     :set             => 248,
@@ -64,6 +70,17 @@ module MysqlBinlog
       a + (b << 8) + (c << 16)
     end
 
+    # Read an unsigned 24-bit (3-byte) big-endian integer.
+    def read_uint24_be
+      a, b = reader.read(3).unpack("nC")
+      (a << 8) + b
+    end
+
+    # Read an unsigned 32-bit (4-byte) integer.
+    def read_uint32_be
+      reader.read(4).unpack("N").first
+    end
+
     # Read an unsigned 32-bit (4-byte) integer.
     def read_uint32
       reader.read(4).unpack("V").first
@@ -75,6 +92,12 @@ module MysqlBinlog
       a + (b << 8)
     end
 
+    # Read an unsigned 40-bit (5-byte) big-endian integer.
+    def read_uint40_be
+      a, b = reader.read(5).unpack("NC")
+      (a << 8) + b
+    end
+
     # Read an unsigned 48-bit (6-byte) integer.
     def read_uint48
       a, b, c = reader.read(6).unpack("vvv")
@@ -89,7 +112,12 @@ module MysqlBinlog
 
     # Read an unsigned 64-bit (8-byte) integer.
     def read_uint64
-      reader.read(8).unpack("Q").first
+      reader.read(8).unpack("Q<").first
+    end
+
+    # Read an unsigned 64-bit (8-byte) integer.
+    def read_uint64_be
+      reader.read(8).unpack("Q>").first
     end
 
     # Read a signed 8-bit (1-byte) integer.
@@ -99,13 +127,13 @@ module MysqlBinlog
 
     # Read a signed 16-bit (2-byte) big-endian integer.
     def read_int16_be
-      reader.read(2).reverse.unpack('s').first
+      reader.read(2).unpack('n').first
     end
 
     # Read a signed 24-bit (3-byte) big-endian integer.
     def read_int24_be
       a, b, c = reader.read(3).unpack('CCC')
-      if a & 128
+      if (a & 128) == 0
         (a << 16) | (b << 8) | c
       else
         (-1 << 24) | (a << 16) | (b << 8) | c
@@ -114,9 +142,9 @@ module MysqlBinlog
 
     # Read a signed 32-bit (4-byte) big-endian integer.
     def read_int32_be
-      reader.read(4).reverse.unpack('l').first
+      reader.read(4).unpack('N').first
     end
-
+    
     def read_uint_by_size(size)
       case size
       when 1
@@ -272,7 +300,7 @@ module MysqlBinlog
         str << value.to_s
       end
 
-      BigDecimal.new(str)
+      BigDecimal(str)
     end
 
     # Read an array of unsigned 8-bit (1-byte) integers.
@@ -359,6 +387,41 @@ module MysqlBinlog
       ]
     end
 
+    def convert_mysql_type_datetimef(int_part, frac_part)
+      year_month = extract_bits(int_part, 17, 22)
+      year = year_month / 13
+      month = year_month % 13
+      day = extract_bits(int_part, 5, 17)
+      hour = extract_bits(int_part, 5, 12)
+      minute = extract_bits(int_part, 6, 6)
+      second = extract_bits(int_part, 6, 0)
+
+      "%04i-%02i-%02i %02i:%02i:%02i.%06i" % [
+        year,
+        month,
+        day,
+        hour,
+        minute,
+        second,
+        frac_part,
+      ]
+    end
+
+    def read_datetimef(decimals)
+      int_part = read_uint40_be
+      frac_part = case decimals
+      when 0
+        0
+      when 1, 2
+        read_uint8 * 10000
+      when 3, 4
+        read_uint16_be * 100
+      when 5, 6
+        read_uint24_be
+      end
+      convert_mysql_type_datetimef(int_part, frac_part)
+    end
+
     # Read a single field, provided the MySQL column type as a symbol. Not all
     # types are currently supported.
     def read_mysql_type(type, metadata=nil)
@@ -382,7 +445,7 @@ module MysqlBinlog
       when :varchar, :string
         prefix_size = (metadata[:max_length] > 255) ? 2 : 1
         read_lpstring(prefix_size)
-      when :blob, :geometry
+      when :blob, :geometry, :json
         read_lpstring(metadata[:length_size])
       when :timestamp
         read_uint32
@@ -394,6 +457,8 @@ module MysqlBinlog
         convert_mysql_type_time(read_uint24)
       when :datetime
         convert_mysql_type_datetime(read_uint64)
+      when :datetime2
+        read_datetimef(metadata[:decimals])
       when :enum, :set
         read_uint_by_size(metadata[:size])
       when :bit

data/lib/mysql_binlog/reader/binlog_file_reader.rb

@@ -20,7 +20,7 @@ module MysqlBinlog
     def open_file(filename)
       @dirname  = File.dirname(filename)
       @filename = File.basename(filename)
-      @binlog   = File.open(filename, mode="r")
+      @binlog   = File.open(filename, "r:BINARY")
 
       verify_magic
     end
@@ -69,7 +69,7 @@ module MysqlBinlog
     end
 
     def remaining(header)
-      header[:next_position] - @binlog.tell
+      header[:payload_end] - @binlog.tell
     end
 
     def skip(header)

data/lib/mysql_binlog/version.rb

@@ -1,3 +1,3 @@
 module MysqlBinlog
-  VERSION = "0.3.2"
+  VERSION = "0.3.3"
 end

metadata

@@ -1,32 +1,23 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mysql_binlog
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: 
-  segments: 
-  - 0
-  - 3
-  - 2
-  version: 0.3.2
+version: !ruby/object:Gem::Version
+  version: 0.3.3
 platform: ruby
-authors: 
+authors:
 - Jeremy Cole
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-11-07 00:00:00 Z
+date: 2019-07-13 00:00:00.000000000 Z
 dependencies: []
-
 description: Library for parsing MySQL binary logs in Ruby
 email: jeremy@jcole.us
-executables: 
+executables:
 - mysql_binlog_dump
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
+- bin/mysql_binlog_dump
 - lib/mysql_binlog.rb
 - lib/mysql_binlog/binlog.rb
 - lib/mysql_binlog/binlog_event_parser.rb
@@ -36,40 +27,27 @@ files:
 - lib/mysql_binlog/reader/binlog_stream_reader.rb
 - lib/mysql_binlog/reader/debugging_reader.rb
 - lib/mysql_binlog/version.rb
-- bin/mysql_binlog_dump
 homepage: http://jcole.us/
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 2.5.2.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: MySQL Binary Log Parser
 test_files: []
-
-has_rdoc: