mysql_binlog 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b81a85e5ebc8ed76b5152df02001e8a2dd6df87
-  data.tar.gz: d74e28a964473c50e1120c3f52fd61333aad8c99
+  metadata.gz: 964c4ea70ef1f90974e39ebe4886a8a1b3496a03
+  data.tar.gz: 489ef416fdd77d92e6c030058761cbf4f063f5fa
 SHA512:
-  metadata.gz: ba05cac9a92cf9e2ad05c37625a25e4d6c506b7ba473b321b8d7536c81e4c299e1060192304fdf8c639abc554b6ed0927fa5ac850d33c8a0a9f4d6ec371e9151
-  data.tar.gz: 1fe544bbc944267de7479316ab0a3a497737e57fe51754cf9e6c1e52e670940097c48780f635d473948b928645705630a19920000f2015fe676bb10758ee7b80
+  metadata.gz: cff1a9268787b0d5f4b74a221545fca87937159a03a6ce3face47871f20fa679036c041cc0b3b3a61c41643a2a056fa1c89acbf81c2622ef4ae11f9e5fb384ce
+  data.tar.gz: eb2838fd4a04e4a5a1307e8144c01bb8bb77ae814e7a48d3b8cc69cc93d9b72aab41382429b2aa3654b4ce14db6561d1b541ad063ca9b8a8304fd939248fdc05

data/bin/mysql_binlog_dump

@@ -6,8 +6,6 @@ require 'getoptlong'
 require 'ostruct'
 require 'pp'
 
-include MysqlBinlog
-
 def usage(exit_code, message = nil)
   print "Error: #{message}\n\n" unless message.nil?
 
@@ -89,11 +87,11 @@ if @options.filenames.empty?
 end
 
 @options.filenames.each do |filename|
-  reader = BinlogFileReader.new(filename)
+  reader = MysqlBinlog::BinlogFileReader.new(filename)
   if @options.debug
-    reader = DebuggingReader.new(reader, :data => true, :calls => true)
+    reader = MysqlBinlog::DebuggingReader.new(reader, :data => true, :calls => true)
   end
-  binlog = Binlog.new(reader)
+  binlog = MysqlBinlog::Binlog.new(reader)
   binlog.checksum = @options.checksum
 
   reader.tail = @options.tail

data/bin/mysql_binlog_summary

@@ -0,0 +1,222 @@
+#!/usr/bin/env ruby
+
+require 'mysql_binlog'
+require 'getoptlong'
+require 'ostruct'
+
+def usage(exit_code, message = nil)
+  print "Error: #{message}\n\n" unless message.nil?
+
+  print <<'END_OF_USAGE'
+
+Usage:
+  To read from a binary log file on disk:
+    mysql_binlog_summary [options] <filename(s)>
+
+  --help, -?
+    Show this help.
+
+  --file, -f <filename>
+    Read from a binary log file on disk (deprecated).
+
+  --checksum, -c
+    Enable CRC32 checksums.
+
+  --tail, -t
+    When reading from a file, follow the end of the binary log file instead
+    of exiting when reaching the end. Exit with Control-C.
+
+  --rotate, -r
+    When reading from a file, follow the rotate events which may be at the
+    end of a file (due to log rotation) so that the stream can be followed
+    through multiple files. This is especially useful with --tail.
+
+END_OF_USAGE
+
+  exit exit_code
+end
+
+@options = OpenStruct.new
+@options.tail = false
+@options.rotate = false
+@options.checksum = nil
+@options.filenames = []
+
+getopt_options = [
+  [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
+  [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--tail",             "-t",     GetoptLong::NO_ARGUMENT ],
+  [ "--rotate",           "-r",     GetoptLong::NO_ARGUMENT ],
+  [ "--checksum",         "-c",     GetoptLong::NO_ARGUMENT ],
+]
+
+getopt = GetoptLong.new(*getopt_options)
+
+getopt.each do |opt, arg|
+  case opt
+  when "--help"
+    usage 0
+  when "--file"
+    @options.filenames << arg
+  when "--tail"
+    @options.tail = true
+  when "--rotate"
+    @options.rotate = true
+  when "--checksum"
+    @options.checksum = :crc32
+  end
+end
+
+@options.filenames.concat(ARGV)
+
+if @options.filenames.empty?
+  usage 1, "A file must be provided"
+end
+
+files = {}
+min_timestamp = nil
+max_timestamp = nil
+events = []
+events_processed = 0
+
+@options.filenames.each do |filename|
+  reader = MysqlBinlog::BinlogFileReader.new(filename)
+  binlog = MysqlBinlog::Binlog.new(reader)
+  reader.tail = @options.tail
+  binlog.ignore_rotate = !@options.rotate
+  binlog.checksum = @options.checksum
+
+  file_min_timestamp = nil
+  file_max_timestamp = nil
+  file_events_processed = 0
+
+  #binlog.filter_event_types = [:query_event]
+  #binlog.filter_flags = [0]
+  query_pattern = /^(INSERT|UPDATE|DELETE)\s+(?:(?:INTO|FROM)\s+)?[`]?(\S+?)[`]?\s+/i
+
+  binlog.each_event do |event|
+    verb  = nil
+    table = nil
+
+    if event[:type] == :query_event
+      if match_query = event[:event][:query].match(query_pattern)
+        verb  = match_query[1].downcase
+        table = match_query[2]
+      end
+    end
+
+    if MysqlBinlog::ROW_EVENT_TYPES.include? event[:type]
+      verb  = event[:type].to_s.sub(/_event_v[12]/, '')
+      table = event[:event][:table][:table]
+    end
+    
+    timestamp = event[:header][:timestamp]
+
+    file_min_timestamp = [file_min_timestamp || timestamp, timestamp].min
+    file_max_timestamp = [file_max_timestamp || timestamp, timestamp].max
+
+    net_change = 0
+    event[:event][:row_image]&.each do |row_image|
+      case verb
+      when "delete_rows"
+        net_change -= row_image[:before][:size]
+      when "update_rows"
+        net_change += row_image[:after][:size] - row_image[:before][:size]
+      when "write_rows"
+        net_change += row_image[:after][:size]
+      end
+    end
+
+    events << {
+      timestamp: timestamp,
+      type: event[:type],
+      verb: verb,
+      table: table,
+      net_change: net_change,
+    }
+
+    file_events_processed += 1
+    events_processed += 1
+
+    if (file_events_processed % 1000) == 0
+      puts "%-32s %6d MiB %10d %10d" % [
+        filename, event[:position]/(1024**2), file_events_processed, events_processed
+      ]
+    end
+  end
+
+  files[filename] = {
+    filename: filename,
+    events: file_events_processed,
+    min_timestamp: file_min_timestamp,
+    max_timestamp: file_max_timestamp,
+  }
+
+  min_timestamp = [min_timestamp || file_min_timestamp, file_min_timestamp].min
+  max_timestamp = [max_timestamp || file_max_timestamp, file_max_timestamp].max
+end
+puts "Done."
+puts
+
+duration = max_timestamp - min_timestamp
+
+puts "File summary:"
+files.each do |filename, file|
+  puts "  %-32s%10s%26s%26s" % [
+    File.basename(filename),
+    file[:events],
+    Time.at(file[:min_timestamp]).utc,
+    Time.at(file[:max_timestamp]).utc,
+  ]
+end
+puts
+
+puts "Summary:"
+puts "  Files:        %d" % [files.size]
+puts "  Events:       %d" % [events_processed]
+puts "  Min Time:     %s" % [Time.at(min_timestamp).utc]
+puts "  Max Time:     %s" % [Time.at(max_timestamp).utc]
+puts "  Duration:     %ds" % [duration]
+puts "  Event Rate:   %0.2f/s" % [events_processed.to_f / duration.to_f]
+puts
+
+events_by_type = Hash.new(0)
+events_by_verb_and_table = {}
+net_change_by_verb_and_table = {}
+net_change_by_table = Hash.new(0)
+events.each do |event|
+  events_by_type[event[:type]] += 1
+  if event[:verb]
+    events_by_verb_and_table[event[:verb]] ||= Hash.new(0)
+    events_by_verb_and_table[event[:verb]][event[:table]] += 1
+    net_change_by_verb_and_table[event[:verb]] ||= Hash.new(0)
+    net_change_by_verb_and_table[event[:verb]][event[:table]] += event[:net_change]
+    net_change_by_table[event[:table]] += event[:net_change]
+  end
+end
+
+puts "Events by type:"
+events_by_type.sort { |a, b| b[1] <=> a[1] }.each do |type, count|
+  puts "  %-50s%10d%10.2f/s" % [type, count, count.to_f / duration.to_f]
+end
+puts
+
+puts "Events by verb and table:"
+events_by_verb_and_table.sort.each do |verb, table_and_count|
+  puts "%s\n" % [verb]
+  table_and_count.sort { |a, b| b[1] <=> a[1] }.each do |table, count|
+    puts "  %-50s%10d%10.2f/s%+10.2f KiB/s" % [
+      table, count, count.to_f / duration.to_f,
+      net_change_by_verb_and_table[verb][table] / 1024.0 / duration.to_f,
+    ]
+  end
+  puts
+end
+
+puts "Net change by table (top 10):"
+net_change_by_table.sort { |a, b| b[1].abs <=> a[1].abs }.first(10).each do |table, net_change|
+  puts "  %-50s%+10.2f KiB/s" % [
+    table, net_change.to_f / 1024.0 / duration.to_f
+  ]
+end
+puts

data/lib/mysql_binlog/version.rb

@@ -1,3 +1,3 @@
 module MysqlBinlog
-  VERSION = "0.3.5"
+  VERSION = "0.3.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mysql_binlog
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Jeremy Cole
@@ -14,10 +14,12 @@ description: Library for parsing MySQL binary logs in Ruby
 email: jeremy@jcole.us
 executables:
 - mysql_binlog_dump
+- mysql_binlog_summary
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/mysql_binlog_dump
+- bin/mysql_binlog_summary
 - lib/mysql_binlog.rb
 - lib/mysql_binlog/binlog.rb
 - lib/mysql_binlog/binlog_event_parser.rb