innodb_ruby 0.7.11 → 0.7.12

data/lib/innodb/list.rb

@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 # An abstract InnoDB "free list" or FLST (renamed to just "list" here as it
 # frequently is used for structures that aren't free lists). This class must
 # be sub-classed to provide an appropriate #object_from_address method.
@@ -11,8 +12,10 @@ class Innodb::List
   # or "NULL" pointer (the page number is UINT32_MAX), or the address if
   # valid.
   def self.get_address(cursor)
-    page    = Innodb::Page.maybe_undefined(cursor.get_uint32)
-    offset  = cursor.get_uint16
+    page    = cursor.name("page") {
+      Innodb::Page.maybe_undefined(cursor.get_uint32)
+    }
+    offset  = cursor.name("offset") { cursor.get_uint16 }
     if page
       {
         :page     => page,
@@ -30,8 +33,8 @@ class Innodb::List
   # linked list.
   def self.get_node(cursor)
     {
-      :prev => get_address(cursor),
-      :next => get_address(cursor),
+      :prev => cursor.name("prev") { get_address(cursor) },
+      :next => cursor.name("next") { get_address(cursor) },
     }
   end
 
@@ -46,9 +49,9 @@ class Innodb::List
   # address.
   def self.get_base_node(cursor)
     {
-      :length => cursor.get_uint32,
-      :first  => get_address(cursor),
-      :last   => get_address(cursor),
+      :length => cursor.name("length") { cursor.get_uint32 },
+      :first  => cursor.name("first") { get_address(cursor) },
+      :last   => cursor.name("last")  { get_address(cursor) },
     }
   end
 
@@ -86,6 +89,11 @@ class Innodb::List
     object_from_address(object.next_address)
   end
 
+  # Return the number of items in the list.
+  def length
+    @base[:length]
+  end
+
   # Return the first object in the list using the list base node "first"
   # address pointer.
   def first
@@ -99,8 +107,16 @@ class Innodb::List
   end
 
   # Return a list cursor for the list.
-  def cursor(node=nil)
-    Cursor.new(self, node)
+  def list_cursor(node=nil)
+    ListCursor.new(self, node)
+  end
+
+  # Return whether the given item is present in the list. This depends on the
+  # item and the items in the list implementing some sufficient == method.
+  # This is implemented rather inefficiently by constructing an array and
+  # leaning on Array#include? to do the real work.
+  def include?(item)
+    each.to_a.include? item
   end
 
   # Iterate through all nodes in the list.
@@ -109,7 +125,7 @@ class Innodb::List
       return enum_for(:each)
     end
 
-    c = cursor
+    c = list_cursor
     while e = c.next
       yield e
     end
@@ -117,7 +133,7 @@ class Innodb::List
 
   # A list iteration cursor used primarily by the Innodb::List #cursor method
   # implicitly. Keeps its own state for iterating through lists efficiently.
-  class Cursor
+  class ListCursor
     def initialize(list, node=nil)
       @list   = list
       @cursor = node
@@ -174,4 +190,20 @@ class Innodb::List::Inode < Innodb::List
       page
     end
   end
-end
+end
+
+class Innodb::List::UndoPage < Innodb::List
+  def object_from_address(address)
+    if address && page = @space.page(address[:page])
+      page
+    end
+  end
+end
+
+class Innodb::List::History < Innodb::List
+  def object_from_address(address)
+    if address && page = @space.page(address[:page])
+      Innodb::UndoLog.new(page, address[:offset] - 34)
+    end
+  end
+end

data/lib/innodb/log.rb

@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 # An InnoDB transaction log file.
 class Innodb::Log
   HEADER_SIZE   = 4 * Innodb::LogBlock::BLOCK_SIZE

data/lib/innodb/log_block.rb

@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 require "innodb/cursor"
 require "pp"
 
@@ -179,4 +180,4 @@ class Innodb::LogBlock
     puts "record:"
     pp record
   end
-end
+end

data/lib/innodb/page.rb

@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 require "innodb/cursor"
 
 # A generic class for any type of page, which handles reading the common
@@ -26,15 +27,24 @@ class Innodb::Page
     # If there is a specialized class available for this page type, re-create
     # the page object using that specialized class.
     if specialized_class = SPECIALIZED_CLASSES[page.type]
-      page = specialized_class.new(space, buffer)
+      page = specialized_class.handle(page, space, buffer)
     end
 
     page
   end
 
+  # Allow the specialized class to do something that isn't 'new' with this page.
+  def self.handle(page, space, buffer)
+    self.new(space, buffer)
+  end
+
   # Initialize a page by passing in a buffer containing the raw page contents.
   # The buffer size should match the space's page size.
   def initialize(space, buffer)
+    unless space && buffer
+      raise "Page can't be initialized from nil space or buffer (space: #{space}, buffer: #{buffer})"
+    end
+
     unless space.page_size == buffer.size
       raise "Buffer size #{buffer.size} is different than space page size"
     end
@@ -56,9 +66,20 @@ class Innodb::Page
     @buffer[offset...(offset + length)]
   end
 
-  # Return an Innodb::Cursor object positioned at a specific offset.
+  # If no block is passed, return an Innodb::Cursor object positioned at a
+  # specific offset. If a block is passed, create a cursor at the provided
+  # offset and yield it to the provided block one time, and then return the
+  # return value of the block.
   def cursor(offset)
-    Innodb::Cursor.new(self, offset)
+    new_cursor = Innodb::Cursor.new(self, offset)
+
+    if block_given?
+      # Call the block once and return its return value.
+      yield new_cursor
+    else
+      # Return the cursor itself.
+      new_cursor
+    end
   end
 
   # Return the byte offset of the start of the "fil" header, which is at the
@@ -83,6 +104,12 @@ class Innodb::Page
     4 + 4
   end
 
+  # Return the position of the "body" of the page, which starts after the FIL
+  # header.
+  def pos_page_body
+    pos_fil_header + size_fil_header
+  end
+
   # InnoDB Page Type constants from include/fil0fil.h.
   PAGE_TYPE = {
     0     => :ALLOCATED,      # Freshly allocated page
@@ -108,23 +135,28 @@ class Innodb::Page
 
   # Return the "fil" header from the page, which is common for all page types.
   def fil_header
-    c = cursor(pos_fil_header)
-    @fil_header ||= {
-      :checksum   => c.get_uint32,
-      :offset     => c.get_uint32,
-      :prev       => Innodb::Page.maybe_undefined(c.get_uint32),
-      :next       => Innodb::Page.maybe_undefined(c.get_uint32),
-      :lsn        => c.get_uint64,
-      :type       => PAGE_TYPE[c.get_uint16],
-      :flush_lsn  => c.get_uint64,
-      :space_id   => c.get_uint32,
-    }
+    @fil_header ||= cursor(pos_fil_header).name("fil") do |c|
+      {
+        :checksum   => c.name("checksum") { c.get_uint32 },
+        :offset     => c.name("offset") { c.get_uint32 },
+        :prev       => c.name("prev") {
+          Innodb::Page.maybe_undefined(c.get_uint32)
+        },
+        :next       => c.name("next") {
+          Innodb::Page.maybe_undefined(c.get_uint32)
+        },
+        :lsn        => c.name("lsn") { c.get_uint64 },
+        :type       => c.name("type") { PAGE_TYPE[c.get_uint16] },
+        :flush_lsn  => c.name("flush_lsn") { c.get_uint64 },
+        :space_id   => c.name("space_id") { c.get_uint32 },
+      }
+    end
   end
 
-  # A helper function to return the page type from the "fil" header, for easier
+  # A helper function to return the checksum from the "fil" header, for easier
   # access.
-  def type
-    fil_header[:type]
+  def checksum
+    fil_header[:checksum]
   end
 
   # A helper function to return the page offset from the "fil" header, for
@@ -152,6 +184,50 @@ class Innodb::Page
     fil_header[:lsn]
   end
 
+  # A helper function to return the page type from the "fil" header, for easier
+  # access.
+  def type
+    fil_header[:type]
+  end
+
+  # Calculate the checksum of the page using InnoDB's algorithm. Two sections
+  # of the page are checksummed separately, and then added together to produce
+  # the final checksum.
+  def calculate_checksum
+    unless size == 16384
+      raise "Checksum calculation is only supported for 16 KiB pages"
+    end
+
+    # Calculate the checksum of the FIL header, except for the following:
+    #   :checksum   (offset 4, size 4)
+    #   :flush_lsn  (offset 26, size 8)
+    #   :space_id   (offset 34, size 4)
+    c_partial_header =
+      Innodb::Checksum.fold_enumerator(
+        cursor(pos_fil_header + 4).each_byte_as_uint8(
+          size_fil_header - 4 - 8 - 4
+        )
+      )
+
+    # Calculate the checksum of the page body, except for the FIL header and
+    # the FIL trailer.
+    c_page_body =
+      Innodb::Checksum.fold_enumerator(
+        cursor(pos_page_body).each_byte_as_uint8(
+          size - size_fil_trailer - size_fil_header
+        )
+      )
+
+    # Add the two checksums together, and mask the result back to 32 bits.
+    (c_partial_header + c_page_body) & Innodb::Checksum::MAX
+  end
+
+  # Is the page corrupt? Calculate the checksum of the page and compare to
+  # the stored checksum; return true or false.
+  def corrupt?
+    checksum != calculate_checksum
+  end
+
   # Implement a custom inspect method to avoid irb printing the contents of
   # the page buffer, since it's very large and mostly not interesting.
   def inspect

data/lib/innodb/page/blob.rb

@@ -0,0 +1,60 @@
+# -*- encoding : utf-8 -*-
+
+class Innodb::Page::Blob < Innodb::Page
+  def pos_blob_header
+    pos_fil_header + size_fil_header
+  end
+
+  def size_blob_header
+    4 + 4
+  end
+
+  def pos_blob_data
+    pos_blob_header + size_blob_header
+  end
+
+  def blob_header
+    cursor(pos_blob_header).name("blob_header") do |c|
+      {
+        :length => c.name("length") { c.get_uint32 },
+        :next   => c.name("next") { Innodb::Page.maybe_undefined(c.get_uint32) },
+      }
+    end
+  end
+
+  def blob_data
+    cursor(pos_blob_data).name("blob_data") do |c|
+      c.get_bytes(blob_header[:length])
+    end
+  end
+
+  def dump_hex(string)
+    slice_size = 16
+    bytes = string.split("").map { |s| s.ord }
+    string.split("").each_slice(slice_size).each_with_index do |slice_bytes, slice_count|
+      puts "%08i  %-23s  %-23s  |%-16s|" % [
+        (slice_count * slice_size),
+        slice_bytes[0..8].map { |n| "%02x" % n.ord }.join(" "),
+        slice_bytes[8..16].map { |n| "%02x" % n.ord }.join(" "),
+        slice_bytes.join(""),
+      ]
+    end
+  end
+
+  # Dump the contents of a page for debugging purposes.
+  def dump
+    super
+
+    puts "blob header:"
+    pp blob_header
+    puts
+
+    puts "blob data:"
+    dump_hex(blob_data)
+    puts
+
+    puts
+  end
+end
+
+Innodb::Page::SPECIALIZED_CLASSES[:BLOB] = Innodb::Page::Blob

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 require "innodb/list"
 require "innodb/xdes"
 
@@ -75,26 +76,34 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
   # Read the FSP (filespace) header, which contains a few counters and flags,
   # as well as list base nodes for each list maintained in the filespace.
   def fsp_header
-    c = cursor(pos_fsp_header)
-    @fsp_header ||= {
-      :space_id           => c.get_uint32,
-      :unused             => c.get_uint32,
-      :size               => c.get_uint32,
-      :free_limit         => c.get_uint32,
-      :flags              => self.class.decode_flags(c.get_uint32),
-      :frag_n_used        => c.get_uint32,
-      :free               => Innodb::List::Xdes.new(@space,
-                              Innodb::List.get_base_node(c)),
-      :free_frag          => Innodb::List::Xdes.new(@space,
-                              Innodb::List.get_base_node(c)),
-      :full_frag          => Innodb::List::Xdes.new(@space,
-                              Innodb::List.get_base_node(c)),
-      :first_unused_seg   => c.get_uint64,
-      :full_inodes        => Innodb::List::Inode.new(@space,
-                              Innodb::List.get_base_node(c)),
-      :free_inodes        => Innodb::List::Inode.new(@space,
-                              Innodb::List.get_base_node(c)),
-    }
+    @fsp_header ||= cursor(pos_fsp_header).name("fsp") do |c|
+      {
+        :space_id           => c.name("space_id") { c.get_uint32 },
+        :unused             => c.name("unused") { c.get_uint32 },
+        :size               => c.name("size") { c.get_uint32 },
+        :free_limit         => c.name("free_limit") { c.get_uint32 },
+        :flags              => c.name("flags") { 
+          self.class.decode_flags(c.get_uint32)
+        },
+        :frag_n_used        => c.name("frag_n_used") { c.get_uint32 },
+        :free               => c.name("list[free]") {
+          Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c))
+        },
+        :free_frag          => c.name("list[free_frag]") {
+          Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c))
+        },
+        :full_frag          => c.name("list[full_frag]") {
+          Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c))
+        },
+        :first_unused_seg   => c.name("first_unused_seg") { c.get_uint64 },
+        :full_inodes        => c.name("list[full_inodes]") {
+          Innodb::List::Inode.new(@space, Innodb::List.get_base_node(c))
+        },
+        :free_inodes        => c.name("list[free_inodes]") {
+          Innodb::List::Inode.new(@space, Innodb::List.get_base_node(c))
+        },
+      }
+    end
   end
 
   # Iterate through all lists in the file space.
@@ -117,9 +126,10 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
       return enum_for(:each_xdes)
     end
 
-    c = cursor(pos_xdes_array)
-    entries_in_xdes_array.times do
-      yield Innodb::Xdes.new(self, c)
+    cursor(pos_xdes_array).name("xdes_array") do |c|
+      entries_in_xdes_array.times do |n|
+        yield Innodb::Xdes.new(self, c)
+      end
     end
   end
 
@@ -140,4 +150,4 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
 end
 
 Innodb::Page::SPECIALIZED_CLASSES[:FSP_HDR] = Innodb::Page::FspHdrXdes
-Innodb::Page::SPECIALIZED_CLASSES[:XDES]    = Innodb::Page::FspHdrXdes
+Innodb::Page::SPECIALIZED_CLASSES[:XDES]    = Innodb::Page::FspHdrXdes

data/lib/innodb/page/index.rb

@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 require "innodb/fseg_entry"
 
 # A specialized class for handling INDEX pages, which contain a portion of
@@ -11,6 +12,60 @@ require "innodb/fseg_entry"
 class Innodb::Page::Index < Innodb::Page
   attr_accessor :record_describer
 
+  # The size (in bytes) of the "next" pointer in each record header.
+  RECORD_NEXT_SIZE = 2
+
+  # The size (in bytes) of the bit-packed fields in each record header for
+  # "redundant" record format.
+  RECORD_REDUNDANT_BITS_SIZE = 4
+
+  # Masks for 1-byte record end-offsets within "redundant" records.
+  RECORD_REDUNDANT_OFF1_OFFSET_MASK   = 0x7f
+  RECORD_REDUNDANT_OFF1_NULL_MASK     = 0x80
+
+  # Masks for 2-byte record end-offsets within "redundant" records.
+  RECORD_REDUNDANT_OFF2_OFFSET_MASK   = 0x3fff
+  RECORD_REDUNDANT_OFF2_NULL_MASK     = 0x8000
+  RECORD_REDUNDANT_OFF2_EXTERN_MASK   = 0x4000
+
+  # The size (in bytes) of the bit-packed fields in each record header for
+  # "compact" record format.
+  RECORD_COMPACT_BITS_SIZE = 3
+
+  # Page direction values possible in the page_header's :direction field.
+  PAGE_DIRECTION = {
+    1 => :left,           # Inserts have been in descending order.
+    2 => :right,          # Inserts have been in ascending order.
+    3 => :same_rec,       # Unused by InnoDB.
+    4 => :same_page,      # Unused by InnoDB.
+    5 => :no_direction,   # Inserts have been in random order.
+  }
+
+  # Record types used in the :type field of the record header.
+  RECORD_TYPES = {
+    0 => :conventional,   # A normal user record in a leaf page.
+    1 => :node_pointer,   # A node pointer in a non-leaf page.
+    2 => :infimum,        # The system "infimum" record.
+    3 => :supremum,       # The system "supremum" record.
+  }
+
+  # This record is the minimum record at this level of the B-tree.
+  RECORD_INFO_MIN_REC_FLAG = 1
+
+  # This record has been marked as deleted.
+  RECORD_INFO_DELETED_FLAG = 2
+
+  # The size (in bytes) of the record pointers in each page directory slot.
+  PAGE_DIR_SLOT_SIZE = 2
+
+  # The minimum number of records "owned" by each record with an entry in
+  # the page directory.
+  PAGE_DIR_SLOT_MIN_N_OWNED = 4
+
+  # The maximum number of records "owned" by each record with an entry in
+  # the page directory.
+  PAGE_DIR_SLOT_MAX_N_OWNED = 8
+
   # Return the byte offset of the start of the "index" page header, which
   # immediately follows the "fil" header.
   def pos_index_header
@@ -19,7 +74,7 @@ class Innodb::Page::Index < Innodb::Page
 
   # The size of the "index" header.
   def size_index_header
-    36
+    2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 8 + 2 + 8
   end
 
   # Return the byte offset of the start of the "fseg" header, which immediately
@@ -33,10 +88,27 @@ class Innodb::Page::Index < Innodb::Page
     2 * Innodb::FsegEntry::SIZE
   end
 
-  # Return the byte offset of the start of records within the page (the
-  # position immediately after the page header).
-  def pos_records
-    size_fil_header + size_index_header + size_fseg_header
+  # Return the size of the header for each record.
+  def size_record_header
+    case page_header[:format]
+    when :compact
+      RECORD_NEXT_SIZE + RECORD_COMPACT_BITS_SIZE
+    when :redundant
+      RECORD_NEXT_SIZE + RECORD_REDUNDANT_BITS_SIZE
+    end
+  end
+
+  # The size of the additional data structures in the header of the system
+  # records, which is just 1 byte in redundant format to store the offset
+  # of the end of the field. This is needed specifically here since we need
+  # to be able to calculate the fixed positions of these system records.
+  def size_mum_record_header_additional
+    case page_header[:format]
+    when :compact
+      0 # No additional data is stored in compact format.
+    when :redundant
+      1 # A 1-byte offset for 1 field is stored in redundant format.
+    end
   end
 
   # The size of the data from the supremum or infimum records.
@@ -49,7 +121,9 @@ class Innodb::Page::Index < Innodb::Page
   # page, and represents a record with a "lower value than any possible user
   # record". The infimum record immediately follows the page header.
   def pos_infimum
-    pos_records + size_record_header + size_record_undefined
+    pos_records +
+      size_record_header +
+      size_mum_record_header_additional
   end
 
   # Return the byte offset of the start of the "origin" of the supremum record,
@@ -57,7 +131,18 @@ class Innodb::Page::Index < Innodb::Page
   # page, and represents a record with a "higher value than any possible user
   # record". The supremum record immediately follows the infimum record.
   def pos_supremum
-    pos_infimum + size_record_header + size_record_undefined + size_mum_record
+    pos_infimum +
+      size_record_header +
+      size_mum_record_header_additional +
+      size_mum_record
+  end
+
+  # Return the byte offset of the start of records within the page (the
+  # position immediately after the page header).
+  def pos_records
+    size_fil_header +
+      size_index_header +
+      size_fseg_header
   end
 
   # Return the byte offset of the start of the user records in a page, which
@@ -96,7 +181,7 @@ class Innodb::Page::Index < Innodb::Page
 
   # Return the amount of free space in the page.
   def free_space
-    page_header[:garbage] +
+    page_header[:garbage_size] +
       (size - size_fil_trailer - directory_space - page_header[:heap_top])
   end
 
@@ -116,33 +201,30 @@ class Innodb::Page::Index < Innodb::Page
     data(pos_user_records, page_header[:heap_top] - pos_user_records)
   end
 
-  # Page direction values possible in the page_header's :direction field.
-  PAGE_DIRECTION = {
-    1 => :left,           # Inserts have been in descending order.
-    2 => :right,          # Inserts have been in ascending order.
-    3 => :same_rec,       # Unused by InnoDB.
-    4 => :same_page,      # Unused by InnoDB.
-    5 => :no_direction,   # Inserts have been in random order.
-  }
-
   # Return the "index" header.
   def page_header
-    c = cursor(pos_index_header)
-    @page_header ||= {
-      :n_dir_slots  => c.get_uint16,
-      :heap_top     => c.get_uint16,
-      :n_heap       => ((n_heap = c.get_uint16) & (2**15-1)),
-      :free         => c.get_uint16,
-      :garbage      => c.get_uint16,
-      :last_insert  => c.get_uint16,
-      :direction    => PAGE_DIRECTION[c.get_uint16],
-      :n_direction  => c.get_uint16,
-      :n_recs       => c.get_uint16,
-      :max_trx_id   => c.get_uint64,
-      :level        => c.get_uint16,
-      :index_id     => c.get_uint64,
-      :format       => (n_heap & 1<<15) == 0 ? :redundant : :compact,
-    }
+    @page_header ||= cursor(pos_index_header).name("index") do |c|
+      index = {
+        :n_dir_slots            => c.name("n_dir_slots") { c.get_uint16 },
+        :heap_top               => c.name("heap_top") { c.get_uint16 },
+        :n_heap_format          => c.name("n_heap_format") { c.get_uint16 },
+        :garbage_offset         => c.name("garbage_offset") { c.get_uint16 },
+        :garbage_size           => c.name("garbage_size") { c.get_uint16 },
+        :last_insert_offset     => c.name("last_insert_offset") { c.get_uint16 },
+        :direction              => c.name("direction") { PAGE_DIRECTION[c.get_uint16] },
+        :n_direction            => c.name("n_direction") { c.get_uint16 },
+        :n_recs                 => c.name("n_recs") { c.get_uint16 },
+        :max_trx_id             => c.name("max_trx_id") { c.get_uint64 },
+        :level                  => c.name("level") { c.get_uint16 },
+        :index_id               => c.name("index_id") { c.get_uint64 },
+      }
+      index[:n_heap] = index[:n_heap_format] & (2**15-1)
+      index[:format] = (index[:n_heap_format] & 1<<15) == 0 ?
+        :redundant : :compact
+      index.delete :n_heap_format
+
+      index
+    end
   end
 
   # A helper function to return the page level from the "page" header, for
@@ -162,156 +244,197 @@ class Innodb::Page::Index < Innodb::Page
     self.prev.nil? && self.next.nil?
   end
 
-  # Return the "fseg" header.
-  def fseg_header
-    c = cursor(pos_fseg_header)
-    @fseg_header ||= {
-      :leaf     => Innodb::FsegEntry.get_inode(@space, c),
-      :internal => Innodb::FsegEntry.get_inode(@space, c),
-    }
-  end
-
-  # The size (in bytes) of the bit-packed fields in each record header.
-  RECORD_BITS_SIZE = 3
-
-  # The size (in bytes) of the "next" pointer in each record header.
-  RECORD_NEXT_SIZE = 2
-
-  # The size (in bytes) of the record pointers in each page directory slot.
-  PAGE_DIR_SLOT_SIZE = 2
-
-  # The minimum number of records "owned" by each record with an entry in
-  # the page directory.
-  PAGE_DIR_SLOT_MIN_N_OWNED = 4
-
-  # The maximum number of records "owned" by each record with an entry in
-  # the page directory.
-  PAGE_DIR_SLOT_MAX_N_OWNED = 8
-
-  # Return the size of the header for each record.
-  def size_record_header
-    case page_header[:format]
-    when :compact
-      RECORD_BITS_SIZE + RECORD_NEXT_SIZE
-    when :redundant
-      RECORD_BITS_SIZE + RECORD_NEXT_SIZE + 1
-    end
+  # A helper function to return the offset to the first free record.
+  def garbage_offset
+    page_header && page_header[:garbage_offset]
   end
 
-  # Return the size of a field in the record header for which no description
-  # could be found (but must be skipped anyway).
-  def size_record_undefined
-    case page_header[:format]
-    when :compact
-      0
-    when :redundant
-      1
+  # Return the "fseg" header.
+  def fseg_header
+    @fseg_header ||= cursor(pos_fseg_header).name("fseg") do |c|
+      {
+        :leaf     => c.name("fseg[leaf]") {
+          Innodb::FsegEntry.get_inode(@space, c)
+        },
+        :internal => c.name("fseg[internal]") {
+          Innodb::FsegEntry.get_inode(@space, c)
+        },
+      }
     end
   end
 
-  # Record types used in the :type field of the record header.
-  RECORD_TYPES = {
-    0 => :conventional,   # A normal user record in a leaf page.
-    1 => :node_pointer,   # A node pointer in a non-leaf page.
-    2 => :infimum,        # The system "infimum" record.
-    3 => :supremum,       # The system "supremum" record.
-  }
-
-  # This record is the minimum record at this level of the B-tree.
-  RECORD_INFO_MIN_REC_FLAG = 1
-
-  # This record has been marked as deleted.
-  RECORD_INFO_DELETED_FLAG = 2
-
   # Return the header from a record.
-  def record_header(offset)
-    c = cursor(offset).backward
-    case page_header[:format]
-    when :compact
-      header = {}
-      header[:next] = c.get_sint16
-      bits1 = c.get_uint16
-      header[:type] = RECORD_TYPES[bits1 & 0x07]
-      header[:order] = (bits1 & 0xf8) >> 3
-      bits2 = c.get_uint8
+  def record_header(cursor)
+    origin = cursor.position
+    header = {}
+    cursor.backward.name("header") do |c|
+      case page_header[:format]
+      when :compact
+        # The "next" pointer is a relative offset from the current record.
+        header[:next] = c.name("next") { origin + c.get_sint16 }
+
+        # Fields packed in a 16-bit integer (LSB first):
+        #   3 bits for type
+        #   13 bits for heap_number
+        bits1 = c.name("bits1") { c.get_uint16 }
+        header[:type] = RECORD_TYPES[bits1 & 0x07]
+        header[:heap_number] = (bits1 & 0xf8) >> 3
+      when :redundant
+        # The "next" pointer is an absolute offset within the page.
+        header[:next] = c.name("next") { c.get_uint16 }
+
+        # Fields packed in a 24-bit integer (LSB first):
+        #   1 bit for offset_size (0 = 2 bytes, 1 = 1 byte)
+        #   10 bits for n_fields
+        #   13 bits for heap_number
+        bits1 = c.name("bits1") { c.get_uint24 }
+        header[:offset_size]  = (bits1 & 1) == 0 ? 2 : 1
+        header[:n_fields]     = (bits1 & (((1 << 10) - 1) <<  1)) >>  1
+        header[:heap_number]  = (bits1 & (((1 << 13) - 1) << 11)) >> 11
+      end
+
+      # Fields packed in an 8-bit integer (LSB first):
+      #   4 bits for n_owned
+      #   4 bits for flags
+      bits2 = c.name("bits2") { c.get_uint8 }
       header[:n_owned] = bits2 & 0x0f
       info = (bits2 & 0xf0) >> 4
       header[:min_rec] = (info & RECORD_INFO_MIN_REC_FLAG) != 0
       header[:deleted] = (info & RECORD_INFO_DELETED_FLAG) != 0
-      case header[:type]
-      when :conventional, :node_pointer
-        # The variable-length part of the record header contains a
-        # bit vector indicating NULL fields and the length of each
-        # non-NULL variable-length field.
-        if record_format
-          header[:null_bitmap] = nbmap = record_null_bitmap(c)
-          header[:variable_length] = record_variable_length(c, nbmap)
-        end
+
+      case page_header[:format]
+      when :compact
+        record_header_compact_additional(header, cursor)
+      when :redundant
+        record_header_redundant_additional(header, cursor)
+      end
+    end
+
+    header
+  end
+
+  # Read additional header information from a compact format record header.
+  def record_header_compact_additional(header, cursor)
+    case header[:type]
+    when :conventional, :node_pointer
+      # The variable-length part of the record header contains a
+      # bit vector indicating NULL fields and the length of each
+      # non-NULL variable-length field.
+      if record_format
+        header[:field_nulls] = cursor.name("field_nulls") {
+          record_header_compact_null_bitmap(cursor)
+        }
+        header[:field_lengths], header[:field_externs] =
+          cursor.name("field_lengths_and_externs") {
+            record_header_compact_variable_lengths_and_externs(cursor,
+              header[:field_nulls])
+          }
       end
-      header
-    when :redundant
-      raise "Not implemented"
     end
   end
 
   # Return an array indicating which fields are null.
-  def record_null_bitmap(cursor)
+  def record_header_compact_null_bitmap(cursor)
     fields = (record_format[:key] + record_format[:row])
 
     # The number of bits in the bitmap is the number of nullable fields.
-    size = fields.count do |f| f.nullable end
+    size = fields.count { |f| f.type.nullable? }
 
     # There is no bitmap if there are no nullable fields.
     return nil unless size > 0
 
     # To simplify later checks, expand bitmap to one for each field.
-    bitmap = Array.new(fields.size, false)
+    bitmap = Array.new(fields.last.position + 1, false)
 
     null_bit_array = cursor.get_bit_array(size).reverse!
 
     # For every nullable field, set whether the field is actually null.
     fields.each do |f|
-      bitmap[f.position] = f.nullable ? (null_bit_array.shift == 1) : false
+      bitmap[f.position] = f.type.nullable? ? (null_bit_array.shift == 1) : false
     end
 
     return bitmap
   end
 
-  # Return an array containing the length of each variable-length field.
-  def record_variable_length(cursor, null_bitmap)
+  # Return an array containing an array of the length of each variable-length
+  # field and an array indicating which fields are stored externally.
+  def record_header_compact_variable_lengths_and_externs(cursor, null_bitmap)
     fields = (record_format[:key] + record_format[:row])
 
-    len_array = Array.new(fields.size, 0)
+    len_array = Array.new(fields.last.position + 1, 0)
+    ext_array = Array.new(fields.last.position + 1, false)
 
     # For each non-NULL variable-length field, the record header contains
     # the length in one or two bytes.
     fields.each do |f|
-      next if f.fixed_len > 0 or null_bitmap[f.position]
+      next if !f.type.variable? or (null_bitmap && null_bitmap[f.position])
 
       len = cursor.get_uint8
+      ext = false
 
       # Two bytes are used only if the length exceeds 127 bytes and the
-      # maximum length exceeds 255 bytes.
-      if len > 127 and f.variable_len > 255
+      # maximum length exceeds 255 bytes (or the field is a BLOB type).
+      if len > 127 && (f.type.blob? || f.type.length > 255)
+        ext = (0x40 & len) != 0
         len = ((len & 0x3f) << 8) + cursor.get_uint8
       end
 
       len_array[f.position] = len
+      ext_array[f.position] = ext
     end
 
-    return len_array
+    return len_array, ext_array
+  end
+
+  # Read additional header information from a redundant format record header.
+  def record_header_redundant_additional(header, cursor)
+    header[:field_lengths] = []
+    header[:field_nulls] = []
+    header[:field_externs] = []
+
+    field_offsets = record_header_redundant_field_end_offsets(header, cursor)
+
+    this_field_offset = 0
+    field_offsets.each do |n|
+      case header[:offset_size]
+      when 1
+        next_field_offset = (n & RECORD_REDUNDANT_OFF1_OFFSET_MASK)
+        header[:field_lengths]  << (next_field_offset - this_field_offset)
+        header[:field_nulls]    << ((n & RECORD_REDUNDANT_OFF1_NULL_MASK) != 0)
+        header[:field_externs]  << false
+      when 2
+        next_field_offset = (n & RECORD_REDUNDANT_OFF2_OFFSET_MASK)
+        header[:field_lengths]  << (next_field_offset - this_field_offset)
+        header[:field_nulls]    << ((n & RECORD_REDUNDANT_OFF2_NULL_MASK) != 0)
+        header[:field_externs]  << ((n & RECORD_REDUNDANT_OFF2_EXTERN_MASK) != 0)
+      end
+      this_field_offset = next_field_offset
+    end
+  end
+
+  # Read field end offsets from the provided cursor for each field as counted
+  # by n_fields.
+  def record_header_redundant_field_end_offsets(header, cursor)
+    (0...header[:n_fields]).to_a.inject([]) do |offsets, n|
+      cursor.name("field_end_offset[#{n}]") {
+        offsets << cursor.get_uint_by_size(header[:offset_size])
+      }
+      offsets
+    end
   end
 
   # Parse and return simple fixed-format system records, such as InnoDB's
   # internal infimum and supremum records.
   def system_record(offset)
-    header = record_header(offset)
-    {
-      :offset => offset,
-      :header => header,
-      :next => offset + header[:next],
-      :data => cursor(offset).get_bytes(size_mum_record),
-    }
+    cursor(offset).name("record[#{offset}]") do |c|
+      header = c.peek { record_header(c) }
+      {
+        :offset => offset,
+        :header => header,
+        :next => header[:next],
+        :data => c.name("data") { c.get_bytes(size_mum_record) },
+      }
+    end
   end
 
   # Return the infimum record on a page.
@@ -328,18 +451,23 @@ class Innodb::Page::Index < Innodb::Page
   def make_record_description
     description = record_describer.cursor_sendable_description(self)
 
-    fields = []
+    position = 0
+    fields = {:type => description[:type], :key => [], :row => []}
 
-    (description[:key] + description[:row]).each_with_index do |d, p|
-      fields << Innodb::Field.new(p, *d)
+    description[:key].each do |d|
+      fields[:key] << Innodb::Field.new(position, *d)
+      position += 1
     end
 
-    n = description[:key].size
+    # Account for TRX_ID and ROLL_PTR.
+    position += 2
 
-    description[:key] = fields.slice(0 .. n-1)
-    description[:row] = fields.slice(n ..  -1)
+    description[:row].each do |d|
+      fields[:row] << Innodb::Field.new(position, *d)
+      position += 1
+    end
 
-    return description
+    fields
   end
 
   # Return (and cache) the record format provided by an external class.
@@ -355,62 +483,75 @@ class Innodb::Page::Index < Innodb::Page
     return infimum  if offset == pos_infimum
     return supremum if offset == pos_supremum
 
-    c = cursor(offset).forward
-
-    # There is a header preceding the row itself, so back up and read it.
-    header = record_header(offset)
-
-    this_record = {
-      :format => page_header[:format],
-      :offset => offset,
-      :header => header,
-      :next => header[:next] == 0 ? nil : (offset + header[:next]),
-    }
-
-    if record_format
-      this_record[:type] = record_format[:type]
+    cursor(offset).forward.name("record[#{offset}]") do |c|
+      # There is a header preceding the row itself, so back up and read it.
+      header = c.peek { record_header(c) }
+
+      this_record = {
+        :format => page_header[:format],
+        :offset => offset,
+        :header => header,
+        :next => header[:next] == 0 ? nil : (header[:next]),
+      }
+
+      if record_format
+        this_record[:type] = record_format[:type]
+
+        # Read the key fields present in all types of pages.
+        this_record[:key] = []
+        this_record[:key_ext] = []
+        c.name("key") do
+          record_format[:key].each do |f|
+            this_record[:key].push f.read(this_record, c)
+            this_record[:key_ext].push f.read_extern(this_record, c)
+          end
+        end
 
-      # Read the key fields present in all types of pages.
-      this_record[:key] = []
-      record_format[:key].each do |f|
-        this_record[:key].push f.read(this_record, c)
-      end
+        # If this is a leaf page of the clustered index, read InnoDB's internal
+        # fields, a transaction ID and roll pointer.
+        if level == 0 && record_format[:type] == :clustered
+          this_record[:transaction_id] = c.name("transaction_id") { c.get_hex(6) }
+          c.name("roll_pointer") do
+            rseg_id_insert_flag = c.name("rseg_id_insert_flag") { c.get_uint8 }
+            this_record[:roll_pointer]   = {
+              :is_insert  => (rseg_id_insert_flag & 0x80) == 0x80,
+              :rseg_id    => rseg_id_insert_flag & 0x7f,
+              :undo_log   => c.name("undo_log") {
+                {
+                  :page   => c.name("page")   { c.get_uint32 },
+                  :offset => c.name("offset") { c.get_uint16 },
+                }
+              }
+            }
+          end
+        end
 
-      # If this is a leaf page of the clustered index, read InnoDB's internal
-      # fields, a transaction ID and roll pointer.
-      if level == 0 && record_format[:type] == :clustered
-        this_record[:transaction_id] = c.get_hex(6)
-        first_byte = c.get_uint8
-        this_record[:roll_pointer]   = {
-          :is_insert  => (first_byte & 0x80) == 0x80,
-          :rseg_id    => first_byte & 0x7f,
-          :undo_log => {
-            :page       => c.get_uint32,
-            :offset     => c.get_uint16,
-          }
-        }
-      end
+        # If this is a leaf page of the clustered index, or any page of a
+        # secondary index, read the non-key fields.
+        if (level == 0 && record_format[:type] == :clustered) ||
+          (record_format[:type] == :secondary)
+          # Read the non-key fields.
+          this_record[:row] = []
+          this_record[:row_ext] = []
+          c.name("row") do
+            record_format[:row].each do |f|
+              this_record[:row].push f.read(this_record, c)
+              this_record[:row_ext].push f.read_extern(this_record, c)
+            end
+          end
+        end
 
-      # If this is a leaf page of the clustered index, or any page of a
-      # secondary index, read the non-key fields.
-      if (level == 0 && record_format[:type] == :clustered) ||
-        (record_format[:type] == :secondary)
-        # Read the non-key fields.
-        this_record[:row] = []
-        record_format[:row].each do |f|
-          this_record[:row].push f.read(this_record, c)
+        # If this is a node (non-leaf) page, it will have a child page number
+        # (or "node pointer") stored as the last field.
+        if level > 0
+          # Read the node pointer in a node (non-leaf) page.
+          this_record[:child_page_number] =
+            c.name("child_page_number") { c.get_uint32 }
         end
       end
 
-      # If this is a node (non-leaf) page, it will have a child page number
-      # (or "node pointer") stored as the last field.
-      if level > 0
-        # Read the node pointer in a node (non-leaf) page.
-        this_record[:child_page_number] = c.get_uint32
-      end
+      this_record
     end
-
-    this_record
   end
 
   # A class for cursoring through records starting from an arbitrary point.
@@ -428,7 +569,13 @@ class Innodb::Page::Index < Innodb::Page
       record = @page.record(@offset)
 
       if record == @page.supremum
+        # We've reached the end of the linked list at supremum.
+        @offset = nil
+      elsif record[:next] == @offset
+        # The record links to itself; go ahead and return it (once), but set
+        # the next offset to nil to end the loop.
         @offset = nil
+        record
       else
         @offset = record[:next]
         record
@@ -462,6 +609,24 @@ class Innodb::Page::Index < Innodb::Page
     nil
   end
 
+  def each_garbage_record
+    unless block_given?
+      return enum_for(:each_garbage_record)
+    end
+
+    if garbage_offset == 0
+      return nil
+    end
+
+    c = record_cursor(garbage_offset)
+
+    while rec = c.record
+      yield rec
+    end
+
+    nil
+  end
+
   # Iterate through all child pages of a node (non-leaf) page, which are
   # stored as records with the child page number as the last field in the
   # record.
@@ -484,9 +649,10 @@ class Innodb::Page::Index < Innodb::Page
     return @directory if @directory
 
     @directory = []
-    c = cursor(pos_directory).backward
-    directory_slots.times do
-      @directory.push c.get_uint16
+    cursor(pos_directory).backward.name("page_directory") do |c|
+      directory_slots.times do |n|
+        @directory.push c.name("slot[#{n}]") { c.get_uint16 }
+      end
     end
 
     @directory
@@ -517,18 +683,26 @@ class Innodb::Page::Index < Innodb::Page
     ]
     puts
 
+    puts "page directory:"
+    pp directory
+    puts
+
     puts "system records:"
     pp infimum
     pp supremum
     puts
 
-    puts "page directory:"
-    pp directory
+    puts "garbage records:"
+    each_garbage_record do |rec|
+      pp rec
+      puts
+    end
     puts
 
     puts "records:"
     each_record do |rec|
       pp rec
+      puts
     end
     puts
   end