innodb_ruby 0.9.0 → 0.9.5

data/bin/innodb_log

@@ -6,52 +6,101 @@ require "ostruct"
 require "set"
 require "innodb"
 
-def log_summary(log, space_ids)
-  puts "%-10s%-30s%-10s%-10s" % [
+def log_summary(log_group)
+  puts "%-20s%-15s%-10s%-12s%-10s" % [
+    "lsn",
     "block",
+    "length",
+    "first_rec",
+    "checkpoint",
+  ]
+  lsn = log_group.start_lsn.first
+  log_group.each_block do |block_index, block|
+    header = block.header
+    puts "%-20i%-15i%-10i%-12i%-10i" % [
+      lsn,
+      header[:block_number],
+      header[:data_length],
+      header[:first_rec_group],
+      header[:checkpoint_no],
+    ]
+    block.dump if @options.dump
+    lsn += Innodb::LogBlock::BLOCK_SIZE
+  end
+end
+
+def log_reader_record_summary(reader, follow)
+  puts "%-10s%-10s%-20s%-10s%-10s%-10s" % [
+    "time",
+    "lsn",
     "type",
+    "size",
     "space",
-    "page",
+    "page"
   ]
-  log.each_block do |block_index, block|
-    record = block.first_record_preamble
-    if record
-      space_id = record[:space]
-      if @options.space_ids.empty? or @options.space_ids.include?(space_id)
-        puts "%-10i%-30s%-10i%-10i" % [
-          block_index,
-          record[:type],
-          space_id,
-          record[:page_number],
-        ]
-        if @options.dump
-          block.dump
-        end
-      end
-    end
+
+  reader.each_record(follow) do |rec|
+    preamble = rec.preamble.dup
+    preamble.default = ""
+    puts "%-10s%-10i%-20s%-10i%-10s%-10s" % [
+      (Time.now.strftime "%H:%M:%S"),
+      rec.lsn.first,
+      preamble[:type].to_s,
+      rec.size,
+      rec.preamble[:space].to_s,
+      rec.preamble[:page_number].to_s,
+    ]
   end
 end
 
+def log_follow_tail_summary(log_group)
+  reader = log_group.reader(log_group.max_checkpoint_lsn)
+  reader.checksum = true
+  log_reader_record_summary(reader, true)
+end
+
+def log_record_summary(log_group, lsn_no)
+  reader = log_group.reader(log_group.start_lsn)
+  reader.seek(lsn_no)
+  log_reader_record_summary(reader, false)
+end
+
+def record_dump(log_group, lsn)
+  log_group.record(lsn).dump
+end
+
 def usage(exit_code, message = nil)
   print "Error: #{message}\n" unless message.nil?
 
   print <<'END_OF_USAGE'
 
-Usage: innodb_log [-d] [-s] <file> [<mode>]
+Usage: innodb_log [-d] [-s] -f <log file> <mode>
 
   --help, -?
     Print this usage text.
 
+  --log-file, -f
+    Load a InnoDB redo log file. Repeat for each log file in the log group.
+
   --dump-blocks, -d
     Dump block header, trailer, and record.
 
-  --space-id, -s
-    Print only given space id (may provide multiple times).
+  --lsn <number>, -l
+    Use the log sequence number <number>.
 
 The following modes are supported:
 
-  log-summary (default)
-    A summary of all blocks within log that contain at least one record.
+  log-summary
+    A summary of each block within the logs.
+
+  log-follow-tail-summary
+    Follow the tail of the log and print a summary of each newly logged record.
+
+  log-record-summary
+    Print a summary of each log record following a given LSN.
+
+  record-dump
+    Dump the contents of a log record, using the Ruby pp ("pretty-print") module.
 
 END_OF_USAGE
 
@@ -59,13 +108,15 @@ END_OF_USAGE
 end
 
 @options = OpenStruct.new
+@options.log_files = []
 @options.dump = false
-@options.space_ids = Set.new
+@options.lsn = nil
 
 getopt_options = [
   [ "--help",                   "-?",     GetoptLong::NO_ARGUMENT ],
+  [ "--log-file",               "-f",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--dump-blocks",            "-d",     GetoptLong::NO_ARGUMENT ],
-  [ "--space-id",               "-s",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--lsn",                    "-l",     GetoptLong::REQUIRED_ARGUMENT ],
 ]
 
 getopt = GetoptLong.new(*getopt_options)
@@ -74,25 +125,40 @@ getopt.each do |opt, arg|
   case opt
   when "--help"
     usage 0
+  when "--log-file"
+    @options.log_files << arg
   when "--dump-blocks"
     @options.dump = true
-  when "--space-id"
-    @options.space_ids << Integer(arg)
+  when "--lsn"
+    @options.lsn = arg.to_i
   end
 end
 
-filename, mode = ARGV.shift(2)
-if filename.nil?
-  usage 1
+mode = ARGV.shift
+
+unless mode
+  usage 1, "At least one mode must be provided"
+end
+
+if @options.log_files.empty?
+  usage 1, "At least one log file (-f) must be specified"
 end
 
-log = Innodb::Log.new(filename)
+if /^(log-)?record-/.match(mode) and !@options.lsn
+  usage 1, "LSN must be specified using -l/--lsn"
+end
 
-mode ||= "log-summary"
+log_group = Innodb::LogGroup.new(@options.log_files.sort)
 
 case mode
 when "log-summary"
-  log_summary(log, @options.space_ids)
+  log_summary(log_group)
+when "log-follow-tail-summary"
+  log_follow_tail_summary(log_group)
+when "log-record-summary"
+  log_record_summary(log_group, @options.lsn)
+when "record-dump"
+  record_dump(log_group, @options.lsn)
 else
   usage 1, "Unknown mode: #{mode}"
 end

data/bin/innodb_space

@@ -5,6 +5,84 @@ require "getoptlong"
 require "ostruct"
 require "innodb"
 
+# Convert a floating point RGB array into an ANSI color number approximating it.
+def rgb_to_ansi(rgb)
+  rgb_n = rgb.map { |c| (c * 5.0).round }
+  16 + (rgb_n[0] * 36) + (rgb_n[1] * 6) + rgb_n[2]
+end
+
+# Interpolate intermediate float-arrays between two float-arrays. Do not
+# include the points a and b in the result.
+def interpolate(a, b, count)
+  deltas = a.each_index.map { |i| b[i] - a[i] }
+  steps = a.each_index.map { |i| deltas[i].to_f / (count.to_f + 1) }
+
+  count.times.to_a.map { |i| a.each_index.map { |j| a[j] + ((i+1).to_f * steps[j]) } }
+end
+
+# Interpolate intermediate float-arrays between each step in a sequence of
+# float-arrays. Include each step in the sequence.
+def interpolate_sequence(sequence, count)
+  result = []
+  result << sequence.first
+  (sequence.size-1).times.map { |n| [sequence[n], sequence[n+1]] }.each do |from, to|
+    interpolate(from, to, count).each do |step|
+      result << step
+    end
+    result << to
+  end
+
+  result
+end
+
+# The RGB values of the typical heatmap progression.
+HEATMAP_PROGRESSION = [
+  [0.0, 0.0, 0.0], # Black
+  [0.0, 0.0, 1.0], # Blue
+  [0.0, 1.0, 1.0], # Cyan
+  [0.0, 1.0, 0.0], # Green
+  [1.0, 1.0, 0.0], # Yellow
+  [1.0, 0.0, 0.0], # Red
+  [1.0, 0.0, 1.0], # Purple
+]
+
+# Typical heatmap color progression.
+ANSI_COLORS_HEATMAP = interpolate_sequence(HEATMAP_PROGRESSION, 6).map { |rgb| rgb_to_ansi(rgb) }
+
+# The 24-step grayscale progression.
+ANSI_COLORS_GRAYSCALE = (0xe8..0xff).to_a
+
+# Return the text supplied with ANSI 256-color coloring applied.
+def ansi_color(color, text)
+  "\x1b[38;5;#{color}m#{text}\x1b[0m"
+end
+
+# Zero and 1/8 through 8/8 illustrations.
+BLOCK_CHARS_V = ["░", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
+BLOCK_CHARS_H = ["░", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"]
+
+# A reasonably large prime number to multiply identifiers by in order to
+# space out the colors used for similar identifiers.
+COLOR_SPACING_PRIME = 999983
+
+# Return a string with a possibly colored filled block for use in printing
+# to an ANSI-capable Unicode-enabled terminal.
+def filled_block(fraction, identifier=nil, block_chars=BLOCK_CHARS_V)
+  if fraction == 0.0
+    block = block_chars[0]
+  else
+    parts = (fraction.to_f * (block_chars.size.to_f-1)).floor
+    block = block_chars[[block_chars.size-1, parts+1].min]
+  end
+  if identifier
+    # ANSI 256-color mode, color palette starts at 10 and contains 216 colors.
+    color = 16 + ((identifier * COLOR_SPACING_PRIME)  % 216)
+    ansi_color(color, block)
+  else
+    block
+  end
+end
+
 # Print metadata about each list in an array of InnoDB::List objects.
 def print_lists(lists)
   puts "%-20s%-12s%-12s%-12s%-12s%-12s" % [
@@ -30,7 +108,7 @@ end
 
 # Print a page usage bitmap for each extent descriptor in an array of
 # Innodb::XdesEntry objects.
-def print_xdes_list(list)
+def print_xdes_list(space, list)
   puts "%-12s%-64s" % [
     "start_page",
     "page_used_bitmap"
@@ -40,7 +118,9 @@ def print_xdes_list(list)
     puts "%-12i%-64s" % [
       entry.xdes[:start_page],
       entry.each_page_status.inject("") { |bitmap, (page_number, page_status)|
-        bitmap += page_status[:free] ? "." : "#"
+        if page_number < space.pages
+          bitmap += page_status[:free] ? "." : "#"
+        end
         bitmap
       },
     ]
@@ -278,7 +358,7 @@ def space_list_iterate(space, list_name)
 
   case fsp[list_name]
   when Innodb::List::Xdes
-    print_xdes_list(fsp[list_name])
+    print_xdes_list(space, fsp[list_name])
   when Innodb::List::Inode
     puts "%-12s" % [
       "page",
@@ -364,7 +444,144 @@ def space_index_pages_free_plot(space, image, start_page)
 end
 
 def space_extents(space)
-  print_xdes_list(space.each_xdes)
+  print_xdes_list(space, space.each_xdes)
+end
+
+# Illustrate the space by printing each extent and for each page, printing a
+# filled block colored based on the index the page is part of. Print a legend
+# for the colors used afterwards.
+def space_extents_illustrate(space)
+  puts
+  puts "%12s ╭%-64s╮" % [ "Start Page", "─" * 64 ]
+
+  identifiers = {}
+
+  space.each_xdes do |entry|
+    puts "%12i │%-64s│" % [
+      entry.xdes[:start_page],
+      entry.each_page_status.inject("") { |bitmap, (page_number, page_status)|
+        if page_number < space.pages
+          used_fraction = 1.0
+          identifier = nil
+          if page_status[:free]
+            used_fraction = 0.0
+          else
+            page = space.page(page_number)
+            if page.respond_to?(:used_space)
+              used_fraction = page.used_space.to_f / page.size.to_f
+            end
+            if page.respond_to?(:index_id)
+              identifier = page.index_id
+              unless identifiers[identifier]
+                identifiers[identifier] = "Index #{page.index_id}"
+                if space.innodb_system
+                  table, index = space.innodb_system.table_and_index_name_by_id(page.index_id)
+                  if table && index
+                    identifiers[identifier] += " (%s.%s)" % [table, index]
+                  end
+                end
+              end
+            end
+          end
+          bitmap += filled_block(used_fraction, identifier)
+        else
+          bitmap += " "
+        end
+        bitmap
+      },
+    ]
+  end
+
+  puts "%12s ╰%-64s╯" % [ "", "─" * 64 ]
+
+  puts
+  puts "Legend:"
+  puts "  %s %s" % [filled_block(1.0, nil), "System"]
+  identifiers.sort.each do |identifier, description|
+    puts "  %s %s" % [filled_block(1.0, identifier), description]
+  end
+  puts "  %s %s" % [filled_block(0.0, nil), "Free space"]
+  puts
+end
+
+def space_lsn_age_illustrate(space)
+  colors = ANSI_COLORS_HEATMAP
+
+  # Calculate the minimum and maximum LSN in the space. This is pretty
+  # inefficient as we end up scanning all pages twice.
+  page_lsn = Array.new(space.pages)
+
+  lsn_min = lsn_max = space.page(0).lsn
+  space.each_page do |page_number, page|
+    if page.lsn != 0
+      page_lsn[page_number] = page.lsn
+      lsn_min = page.lsn < lsn_min ? page.lsn : lsn_min
+      lsn_max = page.lsn > lsn_max ? page.lsn : lsn_max
+    end
+  end
+  lsn_delta = lsn_max - lsn_min
+
+  puts
+  puts "%12s ╭%-64s╮" % [ "Start Page", "─" * 64 ]
+
+  start_page = 0
+  page_lsn.each_slice(64) do |slice|
+    puts "%12i │%-64s│" % [
+      start_page,
+      slice.inject("") { |line, lsn|
+        if lsn
+          age_ratio = (lsn - lsn_min).to_f / lsn_delta.to_f
+          color = colors[(age_ratio * colors.size.to_f).floor]
+          line += ansi_color(color, filled_block(1.0, nil))
+        else
+          line += " "
+        end
+        line
+      },
+    ]
+    start_page += 64
+  end
+
+  puts "%12s ╰%-64s╯" % [ "", "─" * 64 ]
+
+  lsn_legend = "<" + ("─" * (colors.size - 2)) + ">"
+
+  begin
+    # Try to optionally replace the boring lsn_legend with a histogram of
+    # page age distribution. If histogram/array is not available, move on.
+
+    require 'histogram/array'
+
+    lsn_bins, lsn_freq = page_lsn.select { |lsn| !lsn.nil? }.
+      histogram(colors.size, :min => lsn_min, :max => lsn_max)
+
+    lsn_freq_delta = lsn_freq.max - lsn_freq.min
+
+    lsn_legend = ""
+    lsn_freq.each do |freq|
+      freq_norm = freq / lsn_freq_delta
+      if freq_norm > 0.0
+        lsn_legend << filled_block(freq_norm)
+      else
+        # Avoid the "empty" block used for 0.0.
+        lsn_legend << " "
+      end
+    end
+  rescue LoadError
+    # That's okay! Leave the legend boring.
+  end
+
+  puts
+  puts "Legend:"
+  puts "  %12s %s %-12s" % [
+    "Min LSN",
+    lsn_legend,
+    "Max LSN" ]
+  puts "  %12i %s %-12i" % [
+    lsn_min,
+    colors.map { |c| ansi_color(c, filled_block(1.0, nil)) }.join,
+    lsn_max,
+  ]
 end
 
 def print_inode_summary(inode)
@@ -537,6 +754,59 @@ def page_directory_summary(page_number)
   end
 end
 
+def page_illustrate(page)
+  width = 256
+  blocks = Array.new(page.size, " ")
+  identifiers = {}
+  identifier_sort = 0
+
+  page.each_region.sort { |a,b| a[:offset] <=> b[:offset]}.each do |region|
+    region[:length].times do |n|
+      identifier = nil
+      fraction = 0.0
+      if region[:name] != :garbage
+        if n ==  region[:length] - 1
+          fraction = 0.5
+        else
+          fraction = 1.0
+        end
+        identifier = region[:name].hash.abs
+        unless identifiers[identifier]
+          # Prefix an integer <0123> on each name so that the legend can be
+          # sorted by the appearance of each region in the page.
+          identifiers[identifier] = "<%04i>%s" % [
+            identifier_sort,
+            region[:info]
+          ]
+          identifier_sort += 1
+        end
+      end
+      blocks[region[:offset] + n] = filled_block(fraction, identifier, BLOCK_CHARS_H)
+    end
+  end
+
+  puts
+  puts "%12s ╭%-#{width}s╮" % [ "Offset", "─" * width ]
+  offset = 0
+  blocks.each_slice(width) do |slice|
+    puts "%12i │%-s│" % [offset, slice.join]
+    offset += width
+  end
+  puts "%12s ╰%-#{width}s╯" % [ "", "─" * width ]
+
+  puts
+  puts "Legend:"
+  identifiers.sort { |a,b| a[1] <=> b[1] }.each do |identifier, description|
+    puts "  %s %s" % [
+      filled_block(1.0, identifier),
+      description.gsub(/^<\d+>/, "")
+    ]
+  end
+  puts "  %s %s" % [filled_block(0.0, nil), "Garbage"]
+
+  puts
+end
+
 def index_fseg_lists(index, fseg_name)
   unless index.fseg(fseg_name)
     raise "File segment '#{fseg_name}' doesn't exist"
@@ -554,7 +824,7 @@ def index_fseg_list_iterate(index, fseg_name, list_name)
     raise "List '#{list_name}' doesn't exist"
   end
 
-  print_xdes_list(list)
+  print_xdes_list(index.space, list)
 end
 
 def index_fseg_frag_pages(index, fseg_name)
@@ -787,6 +1057,11 @@ The following modes are supported:
   space-extents
     Iterate through all extents, printing the extent descriptor bitmap.
 
+  space-extents-illustrate
+    Iterate through all extents, illustrating the extent usage using ANSI
+    color and Unicode box drawing characters to show page usage throughout
+    the space.
+
   space-inodes-summary
     Iterate through all inodes, printing a short summary of each FSEG.
 
@@ -861,9 +1136,9 @@ getopt_options = [
   [ "--help",                   "-?",     GetoptLong::NO_ARGUMENT ],
   [ "--trace",                  "-t",     GetoptLong::NO_ARGUMENT ],
   [ "--system-space-file",      "-s",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--space-file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--table-name",             "-T",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--index-name",             "-I",     GetoptLong::REQUIRED_ARGUMENT ],
-  [ "--space-file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--page",                   "-p",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--level",                  "-l",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--list",                   "-L",     GetoptLong::REQUIRED_ARGUMENT ],
@@ -879,9 +1154,7 @@ getopt.each do |opt, arg|
   when "--help"
     usage 0
   when "--trace"
-    Innodb::Cursor.trace!
-  when "--mode"
-    @options.mode = arg
+    BufferCursor.trace!
   when "--system-space-file"
     @options.system_space_file = arg
   when "--space-file"
@@ -1012,6 +1285,10 @@ when "space-indexes"
   space_indexes(innodb_system, space)
 when "space-extents"
   space_extents(space)
+when "space-extents-illustrate"
+  space_extents_illustrate(space)
+when "space-lsn-age-illustrate"
+  space_lsn_age_illustrate(space)
 when "space-inodes-summary"
   space_inodes_summary(space)
 when "space-inodes-detail"
@@ -1042,6 +1319,8 @@ when "page-account"
   page_account(innodb_system, space, @options.page)
 when "page-directory-summary"
   page_directory_summary(@options.page)
+when "page-illustrate"
+  page_illustrate(space.page(@options.page))
 else
   usage 1, "Unknown mode: #{mode}"
 end