innodb_ruby 0.6.6 → 0.7.1

data/README.md

@@ -0,0 +1,12 @@
+# A parser for InnoDB file formats, in Ruby #
+
+The purpose for this library and tools is to expose some otherwise hidden internals of InnoDB. This code is not intended for critical production usage. It is definitely buggy, and it may be dangerous. Neither its internal APIs or its output are considered stable and are subject to change at any time.
+
+It is intended as for a few purposes:
+
+* *As a learning tool.* What better way to improve your understanding of a structure than to implement it in another language?
+* *As a teaching tool.* Using `irb` to interactively investigate the actual structures on disk is invaluable for demonstrating (to yourself or others) what is actually happening.
+* *As an investigative tool.* InnoDB unfortunately doesn't provide enough information about what it is doing or has done with its on-disk storage. How full are pages? Exactly how many records per page? How is the B+tree structured for a particular table? All of these questions can be answered easily with `innodb_ruby`.
+* *As a debugging tool.* While making changes to the structures or behaviors of InnoDB, it is necessary to have tools to expose the results both of the original behavior and the new one, in order to validate that the changes have the desired effect.
+
+Various parts of this library and the tools included may have wildly differing maturity levels, as it is worked on primarily based on immediate needs of the authors.

data/bin/innodb_space

@@ -70,6 +70,122 @@ def space_page_type_regions(space)
   end
 end
 
+def space_lists(space)
+  fsp = space.page(0).fsp_header
+  puts "%-20s%-12s%-12s%-12s%-12s%-12s" % [
+    "name",
+    "length",
+    "f_page",
+    "f_offset",
+    "l_page",
+    "l_offset",
+  ]
+  fsp.each do |key, list|
+    next unless list.is_a?(Innodb::List)
+    puts "%-20s%-12i%-12i%-12i%-12i%-12i" % [
+      key,
+      list.base[:length],
+      list.base[:first] && list.base[:first][:page],
+      list.base[:first] && list.base[:first][:offset],
+      list.base[:last] && list.base[:last][:page],
+      list.base[:last] && list.base[:last][:offset],
+    ]
+  end
+end
+
+def space_indexes(space)
+  puts "%-12s%-12s%-12s%-12s%-12s%-12s" % [
+    "id",
+    "root",
+    "fseg",
+    "used",
+    "allocated",
+    "fill_factor",
+  ]
+
+  space.each_index do |index|
+    [:internal, :leaf].each do |fseg_name|
+      fseg = index.root.fseg_header[fseg_name]
+
+      fragments =
+        fseg[:frag_array].inject(0) { |c, i| c += 1 if i; c }
+
+      used =
+        fragments +
+        fseg[:not_full_n_used] +
+        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:full].base[:length]
+
+      allocated =
+        fragments +
+        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:full].base[:length] +
+        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:not_full].base[:length] +
+        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:free].base[:length]
+
+      puts "%-12i%-12i%-12s%-12i%-12i%-12s" % [
+        index.id,
+        index.root.offset,
+        fseg_name,
+        used,
+        allocated,
+        "%.2f%%" % [100.0 * (used.to_f / allocated.to_f)],
+      ]
+    end
+  end
+end
+
+def print_xdes_list(list)
+  puts "%-12s%-64s" % [
+    "start_page",
+    "bitmap"
+  ]
+  list.each do |entry|
+    puts "%-12i%-64s" % [
+      entry.xdes[:start_page],
+      entry.xdes[:bitmap].bytes.map { |byte|
+        [0, 2, 4, 6].map { |shift|
+          ((byte >> shift) & 1 == 1) ? "." : "#"
+        }
+      }.flatten.join,
+    ]
+  end
+end
+
+def list_summary(space, list_name)
+  fsp = space.page(0).fsp_header
+
+  unless fsp[list_name] && fsp[list_name].is_a?(Innodb::List)
+    raise "List '#{list_name}' doesn't exist"
+  end
+
+  case fsp[list_name]
+  when Innodb::List::Xdes
+    print_xdes_list(fsp[list_name])
+  when Innodb::List::Inode
+    puts "%-12s" % [
+      "page",
+    ]
+    fsp[list_name].each do |page|
+      puts "%-12i" % [
+        page.offset,
+      ]
+    end
+  end
+end
+
+def index_fseg_summary(index, fseg_name, list_name)
+  fseg_header = index.root.fseg_header
+  unless fseg_header[fseg_name]
+    raise "File segment '#{fseg_name}' doesn't exist"
+  end
+
+  fseg = fseg_header[fseg_name]
+  unless fseg[list_name] && fseg[list_name].is_a?(Innodb::List)
+    raise "List '#{list_name}' doesn't exist"
+  end
+
+  print_xdes_list(fseg[list_name])
+end
+
 def space_index_pages_free_plot(space, image)
   unless require "gnuplot"
     raise "Couldn't load gnuplot. Is it installed?"
@@ -225,6 +341,9 @@ Usage: innodb_space -f <file> [-p <page>] [-l <level>] <mode> [<mode>, ...]
   --level, -l <level>
     Operate on the level <level>; may be specified more than once.
 
+  --list, -L <list>
+    Operate  on the list <list>; may be specified more than once.
+
   --require, -r <file>
     Use Ruby's "require" to load the file <file>. This is useful for loading
     classes with record describers.
@@ -255,6 +374,16 @@ The following modes are supported:
     Summarize all contiguous regions of the same page type. This is useful to
     provide an overall view of the space and allocations within it.
 
+  space-lists
+    Print the names of all lists in a space.
+
+  space-indexes
+    Summarize all indexes (actually each segment of the indexes) to show
+    the number of pages used and allocated, and the segment fill factor.
+
+  list-summary
+    Summarize the contents of a list.
+
   index-recurse
     Recurse an index, starting at the root (which must be provided in the first
     --page/-p argument), printing the node pages, node pointers (links), leaf
@@ -269,6 +398,13 @@ The following modes are supported:
     Print a summary of all pages at a given level (provided with the --level/-l
     argument) in an index.
 
+  index-fseg-internal-summary
+  index-fseg-leaf-summary
+    Summarize the file segment (whose name is provided in the first --list/-L
+    argument) for internal or leaf pages for a given index (whose root page
+    is provided in the first --page/-p argument). The lists used for each
+    index are "full", "not_full", and "free".
+
 END_OF_USAGE
 
   exit exit_code
@@ -278,6 +414,7 @@ end
 @options.file   = nil
 @options.pages  = []
 @options.levels = []
+@options.lists  = []
 @options.describer = nil
 
 getopt_options = [
@@ -285,6 +422,7 @@ getopt_options = [
   [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--page",             "-p",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--level",            "-l",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--list",             "-L",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--require",          "-r",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--describer",        "-d",     GetoptLong::REQUIRED_ARGUMENT ],
 ]
@@ -303,6 +441,8 @@ getopt.each do |opt, arg|
     @options.pages << arg.to_i
   when "--level"
     @options.levels << arg.to_i
+  when "--list"
+    @options.lists << arg.to_sym
   when "--require"
     require arg
   when "--describer"
@@ -343,6 +483,14 @@ ARGV.each do |mode|
     space_index_pages_free_plot(space, name)
   when "space-page-type-regions"
     space_page_type_regions(space)
+  when "space-lists"
+    space_lists(space)
+  when "space-indexes"
+    space_indexes(space)
+  when "list-summary"
+    @options.lists.each do |list|
+      list_summary(space, list)
+    end
   when "index-recurse"
     unless space.record_describer
       usage 1, "Record describer necessary for index recursion" 
@@ -377,7 +525,15 @@ ARGV.each do |mode|
     end
 
     index_level_summary(space.index(@options.pages.first), @options.levels)
+  when "index-fseg-leaf-summary"
+    @options.lists.each do |list|
+      index_fseg_summary(space.index(@options.pages.first), :leaf, list)
+    end
+  when "index-fseg-internal-summary"
+    @options.lists.each do |list|
+      index_fseg_summary(space.index(@options.pages.first), :internal, list)
+    end
   else
     usage 1, "Unknown mode: #{mode}"
   end
-end
+end

data/lib/innodb.rb

@@ -8,7 +8,9 @@ require "innodb/page/inode"
 require "innodb/page/index"
 require "innodb/page/trx_sys"
 require "innodb/record_describer"
+require "innodb/field"
 require "innodb/space"
 require "innodb/index"
 require "innodb/log_block"
 require "innodb/log"
+require "innodb/xdes"

data/lib/innodb/cursor.rb

@@ -1,3 +1,5 @@
+require "bindata"
+
 # A cursor to walk through InnoDB data structures to read fields.
 class Innodb::Cursor
   def initialize(buffer, offset)
@@ -60,9 +62,8 @@ class Innodb::Cursor
   end
 
   # Read a number of bytes forwards or backwards from the current cursor
-  # position and adjust the cursor position by that amount, optionally
-  # unpacking the data using the provided type.
-  def read_and_advance(length, type=nil)
+  # position and adjust the cursor position by that amount.
+  def read_and_advance(length)
     data = nil
     #print "data(#{@cursor[0]}..."
     case @direction
@@ -74,7 +75,7 @@ class Innodb::Cursor
       data = @buffer.data(@cursor[0], length)
     end
     #puts "#{@cursor[0]}) = #{data.bytes.map { |n| "%02x" % n }.join}"
-    type ? data.unpack(type).first : data
+    data
   end
 
   # Return raw bytes.
@@ -87,50 +88,67 @@ class Innodb::Cursor
     read_and_advance(length).bytes.map { |c| "%02x" % c }.join
   end
 
-  # Return a big-endian unsigned 8-bit integer.
+  # Read an unsigned 8-bit integer.
   def get_uint8(offset=nil)
     seek(offset)
-    read_and_advance(1, "C")
+    data = read_and_advance(1)
+    BinData::Uint8.read(data)
   end
 
-  # Return a big-endian unsigned 16-bit integer.
+  # Read a big-endian unsigned 16-bit integer.
   def get_uint16(offset=nil)
     seek(offset)
-    read_and_advance(2, "n")
+    data = read_and_advance(2)
+    BinData::Uint16be.read(data)
   end
 
-  # Return a big-endian signed 16-bit integer.
+  # Read a big-endian signed 16-bit integer.
   def get_sint16(offset=nil)
     seek(offset)
-    uint = read_and_advance(2, "n")
-    (uint & 32768) == 0 ? uint : -(uint ^ 65535) - 1
+    data = read_and_advance(2)
+    BinData::Int16be.read(data)
   end
 
-  # Return a big-endian unsigned 24-bit integer.
+  # Read a big-endian unsigned 24-bit integer.
   def get_uint24(offset=nil)
     seek(offset)
-    # Ruby 1.8 doesn't support big-endian 24-bit unpack; unpack as one
-    # 8-bit and one 16-bit big-endian instead.
-    high, low = read_and_advance(3).unpack("nC")
-    (high << 8) | low
+    data = read_and_advance(3)
+    BinData::Uint24be.read(data)
   end
 
-  # Return a big-endian unsigned 32-bit integer.
+  # Read a big-endian unsigned 32-bit integer.
   def get_uint32(offset=nil)
     seek(offset)
-    read_and_advance(4, "N")
+    data = read_and_advance(4)
+    BinData::Uint32be.read(data)
   end
 
-  # Return a big-endian unsigned 64-bit integer.
+  # Read a big-endian unsigned 64-bit integer.
   def get_uint64(offset=nil)
     seek(offset)
-    # Ruby 1.8 doesn't support big-endian quad-word unpack; unpack as two
-    # 32-bit big-endian instead.
-    high, low = read_and_advance(8).unpack("NN")
-    (high << 32) | low
+    data = read_and_advance(8)
+    BinData::Uint64be.read(data)
   end
 
-  # Return an InnoDB-compressed unsigned 32-bit integer.
+  # Read a big-endian unsigned integer given its size in bytes.
+  def get_uint_by_size(size)
+    case size
+    when 1
+      get_uint8
+    when 2
+      get_uint16
+    when 3
+      get_uint24
+    when 4
+      get_uint32
+    when 8
+      get_uint64
+    else
+      raise "Not implemented"
+    end
+  end
+
+  # Read an InnoDB-compressed unsigned 32-bit integer.
   def get_ic_uint32
     flag = peek { get_uint8 }
 
@@ -151,15 +169,59 @@ class Innodb::Cursor
     end
   end
 
-  # Return an InnoDB-munged signed 8-bit integer. (This is only implemented
-  # for positive integers at the moment.)
+  # Read an InnoDB-munged signed 8-bit integer.
   def get_i_sint8
-    get_uint8 ^ (1 << 7)
+    data = read_and_advance(1)
+    BinData::Int8.read(data) ^ (-1 << 7)
+  end
+
+  # Read an InnoDB-munged signed 16-bit integer.
+  def get_i_sint16
+    data = read_and_advance(2)
+    BinData::Int16be.read(data) ^ (-1 << 15)
   end
 
-  # Return an InnoDB-munged signed 64-bit integer. (This is only implemented
-  # for positive integers at the moment.)
+  # Read an InnoDB-munged signed 24-bit integer.
+  def get_i_sint24
+    data = read_and_advance(3)
+    BinData::Int24be.read(data) ^ (-1 << 23)
+  end
+
+  # Read an InnoDB-munged signed 32-bit integer.
+  def get_i_sint32
+    data = read_and_advance(4)
+    BinData::Int32be.read(data) ^ (-1 << 31)
+  end
+
+  # Read an InnoDB-munged signed 64-bit integer.
   def get_i_sint64
-    get_uint64 ^ (1 << 63)
+    data = read_and_advance(8)
+    BinData::Int64be.read(data) ^ (-1 << 63)
+  end
+
+  # Read an InnoDB-munged signed integer given its size in bytes.
+  def get_i_sint_by_size(size)
+    case size
+    when 1
+      get_i_sint8
+    when 2
+      get_i_sint16
+    when 3
+      get_i_sint24
+    when 4
+      get_i_sint32
+    when 8
+      get_i_sint64
+    else
+      raise "Not implemented"
+    end
+  end
+
+  # Read an array of 1-bit integers.
+  def get_bit_array(num_bits)
+    size = (num_bits + 7) / 8
+    data = read_and_advance(size)
+    bit_array = BinData::Array.new(:type => :bit1, :initial_length => size * 8)
+    bit_array.read(data).to_ary
   end
-end
+end

data/lib/innodb/field.rb

@@ -0,0 +1,66 @@
+class Innodb::Field
+  attr_reader :position, :nullable, :fixed_len, :variable_len
+
+  def initialize(position, type, *properties)
+    @position = position
+    @type, @fixed_len, @variable_len = parse_data_type(type.to_s)
+    @nullable = (not properties.include?(:NOT_NULL))
+    @unsigned = properties.include?(:UNSIGNED)
+  end
+
+  # Parse the data type description string of a field.
+  def parse_data_type(data_type)
+    case data_type
+    when /^(tinyint|smallint|mediumint|int|bigint)$/i
+      type = data_type.upcase.to_sym
+      fixed_len = fixed_len_map[type]
+      [type, fixed_len, 0]
+    when /varchar\((\d+)\)$/i
+      [:VARCHAR, 0, $1.to_i]
+    else
+      raise "Data type '#{data_type}' is not supported"
+    end
+  end
+
+  # Maps data type to fixed storage length.
+  def fixed_len_map
+    {
+      :TINYINT    => 1,
+      :SMALLINT   => 2,
+      :MEDIUMINT  => 3,
+      :INT        => 4,
+      :BIGINT     => 8,
+    }
+  end
+
+  # Return whether this field is NULL.
+  def null?(record)
+    case record[:format]
+    when :compact
+      header = record[:header]
+      header[:null_bitmap][@position]
+    end
+  end
+
+  # Return the length of this variable-length field.
+  def get_variable_len(record)
+    case record[:format]
+    when :compact
+      header = record[:header]
+      header[:variable_length][@position]
+    end
+  end
+
+  # Read an InnoDB encoded data field.
+  def read(record, cursor)
+    return :NULL if @nullable and null?(record)
+
+    case @type
+    when :TINYINT, :SMALLINT, :MEDIUMINT, :INT, :BIGINT
+      symbol = @unsigned ? :get_uint_by_size : :get_i_sint_by_size
+      cursor.send(symbol, @fixed_len)
+    when :VARCHAR
+      '\'' + cursor.get_bytes(get_variable_len(record)) + '\''
+    end
+  end
+end