mysql_binlog 0.3.2 → 0.3.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 68751dcb93a315960a1ea2ec299a9f26eef45678d58828ccd423c4140bbb6636
+  data.tar.gz: 2ccb823feaf7f0bc50912ecfb9e08c4f8bb44a14dd4e72c88acb60485ad2a2f6
+SHA512:
+  metadata.gz: 50d4c8cb1385c63e6a74ab825f71691cbefbbe66c73786462d690750bd64c8d919c63aa5d3c564c8b8d46d10741a82abe754f482e77416b3ff5c7e1f368abd41
+  data.tar.gz: 381f7c4302b0a8d0964e4fcd1b3bcf034ca4401bd6fc68e78768ae7a53931a70b36d7ff2486f72464fd6d5598c0b2723ee375ff762b76efb8647776855cb1c4c

data/bin/mysql_binlog_dump

@@ -6,8 +6,6 @@ require 'getoptlong'
 require 'ostruct'
 require 'pp'
 
-include MysqlBinlog
-
 def usage(exit_code, message = nil)
   print "Error: #{message}\n\n" unless message.nil?
 
@@ -15,13 +13,16 @@ def usage(exit_code, message = nil)
 
 Usage:
   To read from a binary log file on disk:
-    mysql_binlog_dump [options] -f <filename>
+    mysql_binlog_dump [options] <filename(s)>
 
   --help, -?
     Show this help.
 
   --file, -f <filename>
-    Read from a binary log file on disk.
+    Read from a binary log file on disk (deprecated).
+
+  --checksum, -c
+    Enable CRC32 checksums.
 
   --debug, -d
     Debug reading from the binary log, showing calls into the reader and the
@@ -45,13 +46,16 @@ end
 
 @options = OpenStruct.new
 @options.file = nil
+@options.checksum = nil
 @options.debug = false
 @options.tail = false
 @options.rotate = false
+@options.filenames = []
 
 getopt_options = [
   [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
   [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--checksum",         "-c",     GetoptLong::NO_ARGUMENT ],
   [ "--debug",            "-d",     GetoptLong::NO_ARGUMENT ],
   [ "--tail",             "-t",     GetoptLong::NO_ARGUMENT ],
   [ "--rotate",           "-r",     GetoptLong::NO_ARGUMENT ],
@@ -64,7 +68,9 @@ getopt.each do |opt, arg|
   when "--help"
     usage 0
   when "--file"
-    @options.file = arg
+    @options.filenames << arg
+  when "--checksum"
+    @options.checksum = :crc32
   when "--debug"
     @options.debug = true
   when "--tail"
@@ -74,34 +80,25 @@ getopt.each do |opt, arg|
   end
 end
 
-unless @options.file
-  usage 1, "A file must be provided with --file/-f"
-end
+@options.filenames.concat(ARGV)
 
-reader = BinlogFileReader.new(@options.file)
-if @options.debug
-  reader = DebuggingReader.new(reader, :data => true, :calls => true)
+if @options.filenames.empty?
+  usage 1, "One or more filenames must be provided"
 end
-binlog = Binlog.new(reader)
 
-if @options.tail
-  reader.tail = true
-else
-  reader.tail = false
-end
+@options.filenames.each do |filename|
+  reader = MysqlBinlog::BinlogFileReader.new(filename)
+  if @options.debug
+    reader = MysqlBinlog::DebuggingReader.new(reader, :data => true, :calls => true)
+  end
+  binlog = MysqlBinlog::Binlog.new(reader)
+  binlog.checksum = @options.checksum
 
-if @options.rotate
-  binlog.ignore_rotate = false
-else
-  binlog.ignore_rotate = true
-end
+  reader.tail = @options.tail
+  binlog.ignore_rotate = !@options.rotate
 
-binlog.each_event do |event|
-  puts "%-30s%-20s%20d" % [
-    event[:type],
-    event[:filename],
-    event[:position],
-  ]
-  #pp event
-  #puts
-end
+  binlog.each_event do |event|
+    pp event
+    puts
+  end
+end

data/bin/mysql_binlog_summary

@@ -0,0 +1,241 @@
+#!/usr/bin/env ruby
+
+require 'mysql_binlog'
+require 'getoptlong'
+require 'ostruct'
+
+def usage(exit_code, message = nil)
+  print "Error: #{message}\n\n" unless message.nil?
+
+  print <<'END_OF_USAGE'
+
+Usage:
+  To read from a binary log file on disk:
+    mysql_binlog_summary [options] <filename(s)>
+
+  --help, -?
+    Show this help.
+
+  --file, -f <filename>
+    Read from a binary log file on disk (deprecated).
+
+  --checksum, -c
+    Enable CRC32 checksums.
+
+  --tail, -t
+    When reading from a file, follow the end of the binary log file instead
+    of exiting when reaching the end. Exit with Control-C.
+
+  --rotate, -r
+    When reading from a file, follow the rotate events which may be at the
+    end of a file (due to log rotation) so that the stream can be followed
+    through multiple files. This is especially useful with --tail.
+
+END_OF_USAGE
+
+  exit exit_code
+end
+
+@options = OpenStruct.new
+@options.tail = false
+@options.rotate = false
+@options.checksum = nil
+@options.filenames = []
+
+getopt_options = [
+  [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
+  [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--tail",             "-t",     GetoptLong::NO_ARGUMENT ],
+  [ "--rotate",           "-r",     GetoptLong::NO_ARGUMENT ],
+  [ "--checksum",         "-c",     GetoptLong::NO_ARGUMENT ],
+]
+
+getopt = GetoptLong.new(*getopt_options)
+
+getopt.each do |opt, arg|
+  case opt
+  when "--help"
+    usage 0
+  when "--file"
+    @options.filenames << arg
+  when "--tail"
+    @options.tail = true
+  when "--rotate"
+    @options.rotate = true
+  when "--checksum"
+    @options.checksum = :crc32
+  end
+end
+
+@options.filenames.concat(ARGV)
+
+if @options.filenames.empty?
+  usage 1, "A file must be provided"
+end
+
+files = {}
+min_timestamp = nil
+max_timestamp = nil
+events = []
+events_processed = 0
+
+@options.filenames.each do |filename|
+  reader = MysqlBinlog::BinlogFileReader.new(filename)
+  binlog = MysqlBinlog::Binlog.new(reader)
+  reader.tail = @options.tail
+  binlog.ignore_rotate = !@options.rotate
+  binlog.checksum = @options.checksum
+
+  file_min_timestamp = nil
+  file_max_timestamp = nil
+  file_events_processed = 0
+
+  #binlog.filter_event_types = [:query_event]
+  #binlog.filter_flags = [0]
+  query_pattern = /^(INSERT|UPDATE|DELETE)\s+(?:(?:INTO|FROM)\s+)?[`]?(\S+?)[`]?\s+/i
+
+  binlog.each_event do |event|
+    verb  = nil
+    table = nil
+
+    if event[:type] == :query_event
+      if match_query = event[:event][:query].match(query_pattern)
+        verb  = match_query[1].downcase
+        table = match_query[2]
+      end
+    end
+
+    if MysqlBinlog::ROW_EVENT_TYPES.include? event[:type]
+      verb  = event[:type].to_s.sub(/_event_v[12]/, '')
+      table = event[:event][:table][:table]
+    end
+    
+    timestamp = event[:header][:timestamp]
+
+    file_min_timestamp = [file_min_timestamp || timestamp, timestamp].min
+    file_max_timestamp = [file_max_timestamp || timestamp, timestamp].max
+
+    net_change = 0
+    event[:event][:row_image]&.each do |row_image|
+      case verb
+      when "delete_rows"
+        net_change -= row_image[:before][:size]
+      when "update_rows"
+        net_change += row_image[:after][:size] - row_image[:before][:size]
+      when "write_rows"
+        net_change += row_image[:after][:size]
+      end
+    end
+
+    events << {
+      timestamp: timestamp,
+      size: event[:header][:payload_length],
+      type: event[:type],
+      verb: verb,
+      table: table,
+      net_change: net_change,
+    }
+
+    file_events_processed += 1
+    events_processed += 1
+
+    if (file_events_processed % 1000) == 0
+      puts "%-32s %6d MiB %10d %10d" % [
+        filename, event[:position]/(1024**2), file_events_processed, events_processed
+      ]
+    end
+  end
+
+  files[filename] = {
+    filename: filename,
+    events: file_events_processed,
+    min_timestamp: file_min_timestamp,
+    max_timestamp: file_max_timestamp,
+  }
+
+  min_timestamp = [min_timestamp || file_min_timestamp, file_min_timestamp].min
+  max_timestamp = [max_timestamp || file_max_timestamp, file_max_timestamp].max
+end
+puts "Done."
+puts
+
+duration = max_timestamp - min_timestamp
+
+puts "File summary:"
+files.each do |filename, file|
+  puts "  %-32s%10s%26s%26s" % [
+    File.basename(filename),
+    file[:events],
+    Time.at(file[:min_timestamp]).utc,
+    Time.at(file[:max_timestamp]).utc,
+  ]
+end
+puts
+
+puts "Summary:"
+puts "  Files:        %d" % [files.size]
+puts "  Events:       %d" % [events_processed]
+puts "  Min Time:     %s" % [Time.at(min_timestamp).utc]
+puts "  Max Time:     %s" % [Time.at(max_timestamp).utc]
+puts "  Duration:     %ds" % [duration]
+puts "  Event Rate:   %0.2f/s" % [events_processed.to_f / duration.to_f]
+puts
+
+events_by_type = Hash.new(0)
+events_by_verb_and_table = {}
+size_by_verb_and_table = {}
+size_by_table = Hash.new(0)
+net_change_by_verb_and_table = {}
+net_change_by_table = Hash.new(0)
+events.each do |event|
+  events_by_type[event[:type]] += 1
+  if event[:verb]
+    events_by_verb_and_table[event[:verb]] ||= Hash.new(0)
+    events_by_verb_and_table[event[:verb]][event[:table]] += 1
+    size_by_verb_and_table[event[:verb]] ||= Hash.new(0)
+    size_by_verb_and_table[event[:verb]][event[:table]] += event[:size]
+    size_by_table[event[:table]] += event[:size]
+    net_change_by_verb_and_table[event[:verb]] ||= Hash.new(0)
+    net_change_by_verb_and_table[event[:verb]][event[:table]] += event[:net_change]
+    net_change_by_table[event[:table]] += event[:net_change]
+  end
+end
+
+puts "Events by type:"
+events_by_type.sort { |a, b| b[1] <=> a[1] }.each do |type, count|
+  puts "  %-50s%10d%10.2f/s" % [type, count, count.to_f / duration.to_f]
+end
+puts
+
+puts "Events by verb and table:"
+events_by_verb_and_table.sort.each do |verb, table_and_count|
+  puts "%s\n" % [verb]
+  puts "  %-50s%10s%14s%14s%14s" % [
+    "", "Count", "Rate/s", "Net (KiB/s)", "Size (KiB/s)"
+  ]
+  table_and_count.sort { |a, b| b[1] <=> a[1] }.each do |table, count|
+    puts "  %-50s%10d%14s%+14.2f%14.2f" % [
+      table, count, "%10.2f/s" % [count.to_f / duration.to_f],
+      net_change_by_verb_and_table[verb][table] / 1024.0 / duration.to_f,
+      size_by_verb_and_table[verb][table] / 1024.0 / duration.to_f,
+    ]
+  end
+  puts
+end
+
+puts "Event payload by table (top 10):"
+size_by_table.sort { |a, b| b[1].abs <=> a[1].abs }.first(10).each do |table, size|
+  puts "  %-50s%+10.2f KiB/s" % [
+    table, size.to_f / 1024.0 / duration.to_f
+  ]
+end
+puts
+
+puts "Net change by table (top 10):"
+net_change_by_table.sort { |a, b| b[1].abs <=> a[1].abs }.first(10).each do |table, net_change|
+  puts "  %-50s%+10.2f KiB/s" % [
+    table, net_change.to_f / 1024.0 / duration.to_f
+  ]
+end
+puts
+

data/lib/mysql_binlog/binlog.rb

@@ -51,6 +51,7 @@ module MysqlBinlog
     attr_accessor :filter_flags
     attr_accessor :ignore_rotate
     attr_accessor :max_query_length
+    attr_accessor :checksum
 
     def initialize(reader)
       @reader = reader
@@ -61,6 +62,7 @@ module MysqlBinlog
       @filter_flags = nil
       @ignore_rotate = false
       @max_query_length = 1048576
+      @checksum = :nil
     end
 
     # Rewind to the beginning of the log, if supported by the reader. The
@@ -81,10 +83,16 @@ module MysqlBinlog
     def read_event_fields(header)
       # Delegate the parsing of the event content to a method of the same name
       # in BinlogEventParser.
-      if event_parser.methods.include? header[:event_type].to_s
+      if event_parser.methods.map(&:to_sym).include? header[:event_type]
         fields = event_parser.send(header[:event_type], header)
       end
 
+      unless fields
+        fields = {
+          payload: reader.read(header[:payload_length]),
+        }
+      end
+
       # Check if we've read past the end of the event. This is normally because
       # of an unsupported substructure in the event causing field misalignment
       # or a bug in the event reader method in BinlogEventParser. This may also
@@ -103,6 +111,19 @@ module MysqlBinlog
     end
     private :read_event_fields
 
+    def checksum_length
+      case @checksum
+      when :crc32
+        4
+      else
+        0
+      end
+    end
+
+    def payload_length(header)
+      @fde ? (header[:event_length] - @fde[:header_length] - checksum_length) : 0
+    end
+
     # Scan events until finding one that isn't rejected by the filter rules.
     # If there are no filter rules, this will return the next event provided
     # by the reader.
@@ -119,6 +140,18 @@ module MysqlBinlog
           return nil
         end
 
+        # Skip the remaining part of the header which might not have been
+        # parsed.
+        if @fde
+          reader.seek(position + @fde[:header_length])
+          header[:payload_length] = payload_length(header)
+          header[:payload_end] = position + @fde[:header_length] + payload_length(header)
+        else
+          header[:payload_length] = 0
+          header[:payload_end] = header[:next_position]
+        end
+
+
         if @filter_event_types
           unless @filter_event_types.include? header[:event_type]
             skip_this_event = true

data/lib/mysql_binlog/binlog_event_parser.rb

@@ -26,11 +26,23 @@ module MysqlBinlog
     :pre_ga_write_rows_event    => 20,  # (deprecated)
     :pre_ga_update_rows_event   => 21,  # (deprecated)
     :pre_ga_delete_rows_event   => 22,  # (deprecated)
-    :write_rows_event           => 23,  #
-    :update_rows_event          => 24,  #
-    :delete_rows_event          => 25,  #
+    :write_rows_event_v1        => 23,  #
+    :update_rows_event_v1       => 24,  #
+    :delete_rows_event_v1       => 25,  #
     :incident_event             => 26,  #
     :heartbeat_log_event        => 27,  #
+    :ignorable_log_event        => 28,  #
+    :rows_query_log_event       => 29,  #
+    :write_rows_event_v2        => 30,  #
+    :update_rows_event_v2       => 31,  #
+    :delete_rows_event_v2       => 32,  #
+    :gtid_log_event             => 33,  #
+    :anonymous_gtid_log_event   => 34,  #
+    :previous_gtids_log_event   => 35,  #
+    :transaction_context_event  => 36,  #
+    :view_change_event          => 37,  #
+    :xa_prepare_log_event       => 38,  #
+
     :table_metadata_event       => 50,  # Only in Twitter MySQL
   }
 
@@ -44,9 +56,12 @@ module MysqlBinlog
   # have an identical structure, this list can be used by other programs to
   # know which events can be treated as row events.
   ROW_EVENT_TYPES = [
-    :write_rows_event,
-    :update_rows_event,
-    :delete_rows_event,
+    :write_rows_event_v1,
+    :update_rows_event_v1,
+    :delete_rows_event_v1,
+    :write_rows_event_v2,
+    :update_rows_event_v2,
+    :delete_rows_event_v2,
   ]
 
   # Values for the +flags+ field that may appear in binary logs. There are
@@ -55,11 +70,14 @@ module MysqlBinlog
   #
   # Defined in sql/log_event.h line ~448
   EVENT_HEADER_FLAGS = {
-    :binlog_in_use   => 0x01, # LOG_EVENT_BINLOG_IN_USE_F
-    :thread_specific => 0x04, # LOG_EVENT_THREAD_SPECIFIC_F
-    :suppress_use    => 0x08, # LOG_EVENT_SUPPRESS_USE_F
-    :artificial      => 0x20, # LOG_EVENT_ARTIFICIAL_F
-    :relay_log       => 0x40, # LOG_EVENT_RELAY_LOG_F
+    :binlog_in_use    => 0x0001, # LOG_EVENT_BINLOG_IN_USE_F
+    :thread_specific  => 0x0004, # LOG_EVENT_THREAD_SPECIFIC_F
+    :suppress_use     => 0x0008, # LOG_EVENT_SUPPRESS_USE_F
+    :artificial       => 0x0020, # LOG_EVENT_ARTIFICIAL_F
+    :relay_log        => 0x0040, # LOG_EVENT_RELAY_LOG_F
+    :ignorable        => 0x0080, # LOG_EVENT_IGNORABLE_F
+    :no_filter        => 0x0100, # LOG_EVENT_NO_FILTER_F
+    :mts_isolate      => 0x0200, # LOG_EVENT_MTS_ISOLATE_F
   }
 
   # A mapping array for all values that may appear in the +status+ field of
@@ -79,8 +97,15 @@ module MysqlBinlog
     :table_map_for_update,      #  9 (Q_TABLE_MAP_FOR_UPDATE_CODE)
     :master_data_written,       # 10 (Q_MASTER_DATA_WRITTEN_CODE)
     :invoker,                   # 11 (Q_INVOKER)
+    :updated_db_names,          # 12 (Q_UPDATED_DB_NAMES)
+    :microseconds,              # 13 (Q_MICROSECONDS)
+    :commit_ts,                 # 14 (Q_COMMIT_TS)
+    :commit_ts2,                # 15
+    :explicit_defaults_for_timestamp, # 16
   ]
 
+  QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS = 254
+
   # A mapping hash for all values that may appear in the +flags2+ field of
   # a query_event.
   #
@@ -152,6 +177,10 @@ module MysqlBinlog
     :complete_rows          => 1 << 3,  # COMPLETE_ROWS_F
   }
 
+  GENERIC_ROWS_EVENT_VH_FIELD_TYPES = [
+    :extra_rows_info,                   # ROWS_V_EXTRAINFO_TAG
+  ]
+
   # Parse binary log events from a provided binary log. Must be driven
   # externally, but handles all the details of parsing an event header
   # and the content of the various event types.
@@ -181,11 +210,13 @@ module MysqlBinlog
     def event_header
       header = {}
       header[:timestamp]      = parser.read_uint32
-      header[:event_type]     = EVENT_TYPES[parser.read_uint8]
+      event_type = parser.read_uint8
+      header[:event_type]     = EVENT_TYPES[event_type] || "unknown_#{event_type}".to_sym
       header[:server_id]      = parser.read_uint32
       header[:event_length]   = parser.read_uint32
       header[:next_position]  = parser.read_uint32
       header[:flags] = parser.read_uint_bitmap_by_size_and_name(2, EVENT_HEADER_FLAGS)
+
       header
     end
 
@@ -212,6 +243,25 @@ module MysqlBinlog
       fields
     end
 
+    def _query_event_status_updated_db_names
+      db_count = parser.read_uint8
+      return nil if db_count == QUERY_EVENT_OVER_MAX_DBS_IN_EVENT_MTS
+
+      db_names = []
+      db_count.times do |n|
+        db_name = ""
+        loop do
+          c = reader.read(1)
+          break if c == "\0"
+          db_name << c
+        end
+        db_names << db_name
+      end
+
+      db_names
+    end
+    private :_query_event_status_updated_db_names
+
     # Parse a dynamic +status+ structure within a query_event, which consists
     # of a status_length (uint16) followed by a number of status variables
     # (determined by the +status_length+) each of which consist of:
@@ -223,7 +273,8 @@ module MysqlBinlog
       status_length = parser.read_uint16
       end_position = reader.position + status_length
       while reader.position < end_position
-        status_type = QUERY_EVENT_STATUS_TYPES[parser.read_uint8]
+        status_type_id = parser.read_uint8
+        status_type = QUERY_EVENT_STATUS_TYPES[status_type_id]
         status[status_type] = case status_type
         when :flags2
           parser.read_uint_bitmap_by_size_and_name(4, QUERY_EVENT_FLAGS2)
@@ -252,6 +303,14 @@ module MysqlBinlog
           parser.read_uint16
         when :table_map_for_update
           parser.read_uint64
+        when :updated_db_names
+          _query_event_status_updated_db_names
+        when :commit_ts
+          parser.read_uint64
+        when :microseconds
+          parser.read_uint24
+        else
+          raise "Unknown status type #{status_type_id}"
         end
       end
 
@@ -334,7 +393,7 @@ module MysqlBinlog
           :precision => parser.read_uint8,
           :decimals  => parser.read_uint8,
         }
-      when :blob, :geometry
+      when :blob, :geometry, :json
         { :length_size => parser.read_uint8 }
       when :string, :var_string
         # The :string type sets a :real_type field to indicate the actual type
@@ -350,6 +409,10 @@ module MysqlBinlog
         else
           { :max_length  => (((metadata >> 4) & 0x300) ^ 0x300) + (metadata & 0x00ff) }
         end
+      when :timestamp2, :datetime2, :time2
+        {
+          :decimals => parser.read_uint8,
+        }
       end
     end
     private :_table_map_event_column_metadata_read
@@ -376,7 +439,7 @@ module MysqlBinlog
       map_entry[:db] = parser.read_lpstringz
       map_entry[:table] = parser.read_lpstringz
       columns = parser.read_varint
-      columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] }
+      columns_type = parser.read_uint8_array(columns).map { |c| MYSQL_TYPES[c] || "unknown_#{c}".to_sym }
       columns_metadata = _table_map_event_column_metadata(columns_type)
       columns_nullable = parser.read_bit_array(columns)
 
@@ -415,8 +478,9 @@ module MysqlBinlog
       fields[:flags] = parser.read_uint16
       fields[:columns] = columns.times.map do |c|
         descriptor_length = parser.read_uint32
+        column_type = parser.read_uint8
         @table_map[table_id][:columns][c][:description] = {
-          :type => MYSQL_TYPES[parser.read_uint8],
+          :type => MYSQL_TYPES[column_type] || "unknown_#{column_type}".to_sym,
           :length => parser.read_uint32,
           :scale => parser.read_uint8,
           :character_set => COLLATION[parser.read_uint16],
@@ -435,23 +499,43 @@ module MysqlBinlog
     # and false values identifies which columns are present.
     def _generic_rows_event_row_image(header, fields, columns_used)
       row_image = []
+      start_position = reader.position
       columns_null = parser.read_bit_array(fields[:table][:columns].size)
       fields[:table][:columns].each_with_index do |column, column_index|
+        #puts "column #{column_index} #{column}: used=#{columns_used[column_index]}, null=#{columns_null[column_index]}"
         if !columns_used[column_index]
           row_image << nil
         elsif columns_null[column_index]
           row_image << { column_index => nil }
         else
+          value = parser.read_mysql_type(column[:type], column[:metadata])
           row_image << {
-            column_index =>
-              parser.read_mysql_type(column[:type], column[:metadata])
+            column_index => value,
           }
         end
       end
-      row_image
+      end_position = reader.position
+
+      {
+        image: row_image,
+        size: end_position-start_position
+      }
     end
     private :_generic_rows_event_row_image
 
+    def diff_row_images(before, after)
+      diff = {}
+      before.each_with_index do |before_column, index|
+        after_column = after[index]
+        before_value = before_column.first[1]
+        after_value = after_column.first[1]
+        if before_value != after_value
+          diff[index] = { before: before_value, after: after_value }
+        end
+      end
+      diff
+    end
+
     # Parse the row images present in a row-based replication row event. This
     # is rather incomplete right now due missing support for many MySQL types,
     # but can parse some basic events.
@@ -461,13 +545,14 @@ module MysqlBinlog
       while reader.position < end_position
         row_image = {}
         case header[:event_type]
-        when :write_rows_event
+        when :write_rows_event_v1, :write_rows_event_v2
           row_image[:after]  = _generic_rows_event_row_image(header, fields, columns_used[:after])
-        when :delete_rows_event
+        when :delete_rows_event_v1, :delete_rows_event_v1
           row_image[:before] = _generic_rows_event_row_image(header, fields, columns_used[:before])
-        when :update_rows_event
+        when :update_rows_event_v1, :update_rows_event_v2
           row_image[:before] = _generic_rows_event_row_image(header, fields, columns_used[:before])
           row_image[:after]  = _generic_rows_event_row_image(header, fields, columns_used[:after])
+          row_image[:diff] = diff_row_images(row_image[:before][:image], row_image[:after][:image])
         end
         row_images << row_image
       end
@@ -483,6 +568,16 @@ module MysqlBinlog
     end
     private :_generic_rows_event_row_images
 
+    # Parse the variable header from a v2 rows event. This is only used for
+    # ROWS_V_EXTRAINFO_TAG which is used by NDB. Ensure it can be skipped
+    # properly but don't bother parsing it.
+    def _generic_rows_event_vh
+      vh_payload_len = parser.read_uint16 - 2
+      return unless vh_payload_len > 0
+
+      reader.read(vh_payload_len)
+    end
+
     # Parse fields for any of the row-based replication row events:
     # * +Write_rows+ which is used for +INSERT+.
     # * +Update_rows+ which is used for +UPDATE+.
@@ -491,29 +586,104 @@ module MysqlBinlog
     # Implemented in sql/log_event.cc line ~8039
     # in Rows_log_event::write_data_header
     # and Rows_log_event::write_data_body
-    def generic_rows_event(header)
+    def _generic_rows_event(header, contains_vh: false)
       fields = {}
       table_id = parser.read_uint48
       fields[:table] = @table_map[table_id]
       fields[:flags] = parser.read_uint_bitmap_by_size_and_name(2, GENERIC_ROWS_EVENT_FLAGS)
+
+      # Rows_log_event v2 events contain a variable-sized header. Only NDB
+      # uses it right now, so let's just make sure it's skipped properly.
+      _generic_rows_event_vh if contains_vh
+
       columns = parser.read_varint
       columns_used = {}
       case header[:event_type]
-      when :write_rows_event
+      when :write_rows_event_v1, :write_rows_event_v2
         columns_used[:after]  = parser.read_bit_array(columns)
-      when :delete_rows_event
+      when :delete_rows_event_v1, :delete_rows_event_v2
         columns_used[:before] = parser.read_bit_array(columns)
-      when :update_rows_event
+      when :update_rows_event_v1, :update_rows_event_v2
         columns_used[:before] = parser.read_bit_array(columns)
         columns_used[:after]  = parser.read_bit_array(columns)
       end
       fields[:row_image] = _generic_rows_event_row_images(header, fields, columns_used)
       fields
     end
+    private :_generic_rows_event
+
+    def generic_rows_event_v1(header)
+      _generic_rows_event(header, contains_vh: false)
+    end
+
+    alias :write_rows_event_v1  :generic_rows_event_v1
+    alias :update_rows_event_v1 :generic_rows_event_v1
+    alias :delete_rows_event_v1 :generic_rows_event_v1
 
-    alias :write_rows_event  :generic_rows_event
-    alias :update_rows_event :generic_rows_event
-    alias :delete_rows_event :generic_rows_event
+    def generic_rows_event_v2(header)
+      _generic_rows_event(header, contains_vh: true)
+    end
+
+    alias :write_rows_event_v2  :generic_rows_event_v2
+    alias :update_rows_event_v2 :generic_rows_event_v2
+    alias :delete_rows_event_v2 :generic_rows_event_v2
+
+    def rows_query_log_event(header)
+      reader.read(1) # skip useless byte length which is unused
+      { query: reader.read(header[:payload_length]-1) }
+    end
+
+    def in_hex(bytes)
+      bytes.each_byte.map { |c| "%02x" % c.ord }.join
+    end
 
+    def format_gtid_sid(sid)
+      [0..3, 4..5, 6..7, 8..9, 10..15].map { |r| in_hex(sid[r]) }.join("-")
+    end
+
+    # 6d9190a2-cca6-11e8-aa8c-42010aef0019:551845019
+    def format_gtid(sid, gno_or_ivs)
+      "#{format_gtid_sid(sid)}:#{gno_or_ivs}"
+    end
+
+    def previous_gtids_log_event(header)
+      n_sids = parser.read_uint64
+
+      gtids = []
+      n_sids.times do
+        sid = parser.read_nstring(16)
+        n_ivs = parser.read_uint64
+        ivs = []
+        n_ivs.times do
+          iv_start = parser.read_uint64
+          iv_end = parser.read_uint64
+          ivs << "#{iv_start}-#{iv_end}"
+        end
+        gtids << format_gtid(sid, ivs.join(":"))
+      end
+
+      {
+        previous_gtids: gtids
+      }
+    end
+
+    def gtid_log_event(header)
+      flags = parser.read_uint8
+      sid = parser.read_nstring(16)
+      gno = parser.read_uint64
+      lts_type = parser.read_uint8
+      lts_last_committed = parser.read_uint64
+      lts_sequence_number = parser.read_uint64
+
+      {
+        flags: flags,
+        gtid: format_gtid(sid, gno),
+        lts: {
+          type: lts_type,
+          last_committed: lts_last_committed,
+          sequence_number: lts_sequence_number,
+        },
+      }
+    end
   end
 end

data/lib/mysql_binlog/binlog_field_parser.rb

@@ -1,3 +1,5 @@
+require 'bigdecimal'
+
 module MysqlBinlog
   # All MySQL types mapping to their integer values.
   MYSQL_TYPES_HASH = {
@@ -18,6 +20,10 @@ module MysqlBinlog
     :newdate         => 14,
     :varchar         => 15,
     :bit             => 16,
+    :timestamp2      => 17,
+    :datetime2       => 18,
+    :time2           => 19,
+    :json            => 245,
     :newdecimal      => 246,
     :enum            => 247,
     :set             => 248,
@@ -58,12 +64,28 @@ module MysqlBinlog
       reader.read(2).unpack("v").first
     end
 
+    # Read an unsigned 16-bit (2-byte) big-endian integer.
+    def read_uint16_be
+      reader.read(2).unpack("n").first
+    end
+
     # Read an unsigned 24-bit (3-byte) integer.
     def read_uint24
       a, b, c = reader.read(3).unpack("CCC")
       a + (b << 8) + (c << 16)
     end
 
+    # Read an unsigned 24-bit (3-byte) big-endian integer.
+    def read_uint24_be
+      a, b = reader.read(3).unpack("nC")
+      (a << 8) + b
+    end
+
+    # Read an unsigned 32-bit (4-byte) integer.
+    def read_uint32_be
+      reader.read(4).unpack("N").first
+    end
+
     # Read an unsigned 32-bit (4-byte) integer.
     def read_uint32
       reader.read(4).unpack("V").first
@@ -75,6 +97,12 @@ module MysqlBinlog
       a + (b << 8)
     end
 
+    # Read an unsigned 40-bit (5-byte) big-endian integer.
+    def read_uint40_be
+      a, b = reader.read(5).unpack("NC")
+      (a << 8) + b
+    end
+
     # Read an unsigned 48-bit (6-byte) integer.
     def read_uint48
       a, b, c = reader.read(6).unpack("vvv")
@@ -89,7 +117,12 @@ module MysqlBinlog
 
     # Read an unsigned 64-bit (8-byte) integer.
     def read_uint64
-      reader.read(8).unpack("Q").first
+      reader.read(8).unpack("Q<").first
+    end
+
+    # Read an unsigned 64-bit (8-byte) integer.
+    def read_uint64_be
+      reader.read(8).unpack("Q>").first
     end
 
     # Read a signed 8-bit (1-byte) integer.
@@ -99,13 +132,13 @@ module MysqlBinlog
 
     # Read a signed 16-bit (2-byte) big-endian integer.
     def read_int16_be
-      reader.read(2).reverse.unpack('s').first
+      reader.read(2).unpack('n').first
     end
 
     # Read a signed 24-bit (3-byte) big-endian integer.
     def read_int24_be
       a, b, c = reader.read(3).unpack('CCC')
-      if a & 128
+      if (a & 128) == 0
         (a << 16) | (b << 8) | c
       else
         (-1 << 24) | (a << 16) | (b << 8) | c
@@ -114,9 +147,9 @@ module MysqlBinlog
 
     # Read a signed 32-bit (4-byte) big-endian integer.
     def read_int32_be
-      reader.read(4).reverse.unpack('l').first
+      reader.read(4).unpack('N').first
     end
-
+    
     def read_uint_by_size(size)
       case size
       when 1
@@ -272,7 +305,7 @@ module MysqlBinlog
         str << value.to_s
       end
 
-      BigDecimal.new(str)
+      BigDecimal(str)
     end
 
     # Read an array of unsigned 8-bit (1-byte) integers.
@@ -359,6 +392,47 @@ module MysqlBinlog
       ]
     end
 
+    def convert_mysql_type_datetimef(int_part, frac_part)
+      year_month = extract_bits(int_part, 17, 22)
+      year = year_month / 13
+      month = year_month % 13
+      day = extract_bits(int_part, 5, 17)
+      hour = extract_bits(int_part, 5, 12)
+      minute = extract_bits(int_part, 6, 6)
+      second = extract_bits(int_part, 6, 0)
+
+      "%04i-%02i-%02i %02i:%02i:%02i.%06i" % [
+        year,
+        month,
+        day,
+        hour,
+        minute,
+        second,
+        frac_part,
+      ]
+    end
+
+    def read_frac_part(decimals)
+      case decimals
+      when 0
+        0
+      when 1, 2
+        read_uint8 * 10000
+      when 3, 4
+        read_uint16_be * 100
+      when 5, 6
+        read_uint24_be
+      end
+    end
+
+    def read_datetimef(decimals)
+      convert_mysql_type_datetimef(read_uint40_be, read_frac_part(decimals))
+    end
+
+    def read_timestamp2(decimals)
+      read_uint32_be + (read_frac_part(decimals) / 1000000)
+    end
+
     # Read a single field, provided the MySQL column type as a symbol. Not all
     # types are currently supported.
     def read_mysql_type(type, metadata=nil)
@@ -382,10 +456,12 @@ module MysqlBinlog
       when :varchar, :string
         prefix_size = (metadata[:max_length] > 255) ? 2 : 1
         read_lpstring(prefix_size)
-      when :blob, :geometry
+      when :blob, :geometry, :json
         read_lpstring(metadata[:length_size])
       when :timestamp
         read_uint32
+      when :timestamp2
+        read_timestamp2(metadata[:decimals])
       when :year
         read_uint8 + 1900
       when :date
@@ -394,6 +470,8 @@ module MysqlBinlog
         convert_mysql_type_time(read_uint24)
       when :datetime
         convert_mysql_type_datetime(read_uint64)
+      when :datetime2
+        read_datetimef(metadata[:decimals])
       when :enum, :set
         read_uint_by_size(metadata[:size])
       when :bit

data/lib/mysql_binlog/reader/binlog_file_reader.rb

@@ -20,7 +20,7 @@ module MysqlBinlog
     def open_file(filename)
       @dirname  = File.dirname(filename)
       @filename = File.basename(filename)
-      @binlog   = File.open(filename, mode="r")
+      @binlog   = File.open(filename, "r:BINARY")
 
       verify_magic
     end
@@ -69,7 +69,7 @@ module MysqlBinlog
     end
 
     def remaining(header)
-      header[:next_position] - @binlog.tell
+      header[:payload_end] - @binlog.tell
     end
 
     def skip(header)

data/lib/mysql_binlog/version.rb

@@ -1,3 +1,3 @@
 module MysqlBinlog
-  VERSION = "0.3.2"
+  VERSION = "0.3.7"
 end

metadata

@@ -1,32 +1,25 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mysql_binlog
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: 
-  segments: 
-  - 0
-  - 3
-  - 2
-  version: 0.3.2
+version: !ruby/object:Gem::Version
+  version: 0.3.7
 platform: ruby
-authors: 
+authors:
 - Jeremy Cole
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-11-07 00:00:00 Z
+date: 2021-04-14 00:00:00.000000000 Z
 dependencies: []
-
 description: Library for parsing MySQL binary logs in Ruby
 email: jeremy@jcole.us
-executables: 
+executables:
 - mysql_binlog_dump
+- mysql_binlog_summary
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
+- bin/mysql_binlog_dump
+- bin/mysql_binlog_summary
 - lib/mysql_binlog.rb
 - lib/mysql_binlog/binlog.rb
 - lib/mysql_binlog/binlog_event_parser.rb
@@ -36,40 +29,27 @@ files:
 - lib/mysql_binlog/reader/binlog_stream_reader.rb
 - lib/mysql_binlog/reader/debugging_reader.rb
 - lib/mysql_binlog/version.rb
-- bin/mysql_binlog_dump
 homepage: http://jcole.us/
-licenses: []
-
+licenses:
+- BSD-3-Clause
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 3.1.4
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: MySQL Binary Log Parser
 test_files: []
-
-has_rdoc: