innodb_ruby 0.6.6 → 0.7.1

data/lib/innodb/fseg_entry.rb

@@ -1,11 +1,23 @@
+# An InnoDB file segment entry, which appears in a few places, such as the
+# FSEG header of INDEX pages, and in the TRX_SYS pages.
 class Innodb::FsegEntry
   SIZE = 4 + 4 + 2
 
-  def self.get_entry(cursor)
+  # Return the FSEG entry address, which points to an entry on an INODE page.
+  def self.get_entry_address(cursor)
     {
       :space_id     => cursor.get_uint32,
       :page_number  => cursor.get_uint32,
       :offset       => cursor.get_uint16,
     }
   end
+
+  # Return an INODE entry which represents this file segment.
+  def self.get_inode(space, cursor)
+    address = get_entry_address(cursor)
+    page = space.page(address[:page_number])
+    if page.type == :INODE
+      page.inode_at(address[:offset])
+    end
+  end
 end

data/lib/innodb/index.rb

@@ -102,12 +102,15 @@ class Innodb::Index
   #   -1 = a is less than b
   #   +1 = a is greater than b
   def compare_key(a, b)
-    return -1 if a.size < b.size
-    return +1 if a.size > b.size
+    return 0 if a.nil? && b.nil?
+    return -1 if a.nil? || (!b.nil? && a.size < b.size)
+    return +1 if b.nil? || (!a.nil? && a.size > b.size)
+
     a.each_index do |i|
       return -1 if a[i] < b[i]
       return +1 if a[i] > b[i]
     end
+
     return 0
   end
 
@@ -116,20 +119,87 @@ class Innodb::Index
   # than the key. (If an exact match is desired, compare_key must be used to
   # check if the returned record matches. This makes the function useful for
   # search in both leaf and non-leaf pages.)
-  def linear_search_in_page(page, key)
-    c = page.record_cursor(page.infimum[:next])
-    this_rec = c.record
-    while next_rec = c.record
-      return this_rec if next_rec == page.supremum
+  def linear_search_from_cursor(cursor, key)
+    this_rec = cursor.record
+
+    # Iterate through all records until finding either a matching record or
+    # one whose key is greater than the desired key.
+    while this_rec && next_rec = cursor.record
+      # If we reach supremum, return the last non-system record we got.
+      return this_rec if next_rec[:header][:type] == :supremum
+
       if (compare_key(key, this_rec[:key]) >= 0) &&
         (compare_key(key, next_rec[:key]) < 0)
+        # The desired key is either an exact match for this_rec or is greater
+        # than it but less than next_rec. If this is a non-leaf page, that
+        # will mean that the record will fall on the leaf page this node
+        # pointer record points to, if it exists at all.
         return this_rec
       end
+
       this_rec = next_rec
     end
+
     this_rec
   end
 
+  # Search or a record within a single page using the page directory to limit
+  # the number of record comparisons required. Once the last page directory
+  # entry closest to but not greater than the key is found, fall back to
+  # linear search using linear_search_from_cursor to find the closest record
+  # whose key is not greater than the desired key. (If an exact match is
+  # desired, the returned record must be checked in the same way as the above
+  # linear_search_from_cursor function.)
+  def binary_search_by_directory(page, dir, key)
+    return nil if dir.empty?
+
+    # Split the directory at the mid-point (using integer math, so the division
+    # is rounding down). Retrieve the record that sits at the mid-point.
+    mid = dir.size / 2
+    rec = page.record(dir[mid])
+
+    # The mid-point record was the infimum record, which is not comparable with
+    # compare_key, so we need to just linear scan from here. If the mid-point
+    # is the beginning of the page there can't be many records left to check
+    # anyway.
+    if rec[:header][:type] == :infimum
+      return linear_search_from_cursor(page.record_cursor(rec[:next]), key)
+    end
+
+    # Compare the desired key to the mid-point record's key.
+    case compare_key(key, rec[:key])
+    when 0
+      # An exact match for the key was found. Return the record.
+      rec
+    when +1
+      # The mid-point record's key is less than the desired key.
+      if dir.size == 1
+        # This is the last entry remaining from the directory, use linear
+        # search to find the record. We already know that there wasn't an
+        # exact match, so skip the current record and start cursoring from
+        # the next record.
+        linear_search_from_cursor(page.record_cursor(rec[:next]), key)
+      else
+        # There are more entries remaining from the directory, recurse again
+        # using binary search on the right half of the directory, which
+        # represents values greater than or equal to the mid-point record's
+        # key.
+        binary_search_by_directory(page, dir[mid...dir.size], key)
+      end
+    when -1
+      # The mid-point record's key is greater than the desired key.
+      if dir.size == 1
+        # If this is the last entry remaining from the directory, we didn't
+        # find anything workable.
+        nil
+      else
+        # Recurse on the left half of the directory, which represents values
+        # less than the mid-point record's key.
+        binary_search_by_directory(page, dir[0...mid], key)
+      end
+    end
+  end
+
   # Search for a record within the entire index, walking down the non-leaf
   # pages until a leaf page is found, and then verifying that the record
   # returned on the leaf page is an exact match for the key. If a matching
@@ -138,13 +208,41 @@ class Innodb::Index
   def linear_search(key)
     page = @root
 
-    while rec = linear_search_in_page(page, key)
+    while rec =
+      linear_search_from_cursor(page.record_cursor(page.infimum[:next]), key)
       if page.level > 0
+        # If we haven't reached a leaf page yet, move down the tree and search
+        # again using linear search.
         page = @space.page(rec[:child_page_number])
       else
+        # We're on a leaf page, so return the page and record if there is a
+        # match. If there is no match, break the loop and cause nil to be
+        # returned.
         return page, rec if compare_key(key, rec[:key]) == 0
         break
       end
     end
   end
+
+  # Search for a record within the entire index like linear_search, but use
+  # the page directory to search while making as few record comparisons as
+  # possible. If a matching record is not found, nil is returned.
+  def binary_search(key)
+    page = @root
+
+    while rec = binary_search_by_directory(page, page.directory.dup, key)
+      if page.level > 0
+        # If we haven't reached a leaf page yet, move down the tree and search
+        # again using binary search.
+        page = @space.page(rec[:child_page_number])
+      else
+        # We're on a leaf page, so return the page and record if there is a
+        # match. If there is no match, break the loop and cause nil to be
+        # returned.
+        return page, rec if compare_key(key, rec[:key]) == 0
+        break
+      end
+    end
+  end
+
 end

data/lib/innodb/list.rb

@@ -0,0 +1,109 @@
+class Innodb::List
+  FIL_ADDR_SIZE   = 4 + 2
+  NODE_SIZE       = 2 * FIL_ADDR_SIZE
+  BASE_NODE_SIZE  = 4 + (2 * FIL_ADDR_SIZE)
+
+  def self.get_address(cursor)
+    page    = Innodb::Page.maybe_undefined(cursor.get_uint32)
+    offset  = cursor.get_uint16
+    if page
+      {
+        :page     => page,
+        :offset   => offset,
+      }
+    end
+  end
+
+  def self.get_node(cursor)
+    {
+      :prev => get_address(cursor),
+      :next => get_address(cursor),
+    }
+  end
+
+  def self.get_base_node(cursor)
+    {
+      :length => cursor.get_uint32,
+      :first  => get_address(cursor),
+      :last   => get_address(cursor),
+    }
+  end
+
+  def initialize(space, base)
+    @space  = space
+    @base   = base
+  end
+
+  attr_reader :space
+  attr_reader :base
+
+  def prev(object)
+    object_from_address(object.prev_address)
+  end
+
+  def next(object)
+    object_from_address(object.next_address)
+  end
+
+  def first
+    object_from_address(@base[:first])
+  end
+
+  def last
+    object_from_address(@base[:last])
+  end
+
+  def cursor(node=nil)
+    Cursor.new(self, node)
+  end
+
+  def each
+    c = cursor
+    while e = c.next
+      yield e
+    end
+  end
+
+  class Cursor
+    def initialize(list, node=nil)
+      @list   = list
+      @cursor = node
+    end
+
+    def reset
+      @cursor = nil
+    end
+
+    def prev
+      if @cursor
+        @cursor = @list.prev(@cursor)
+      else
+        @cursor = @list.last
+      end
+    end
+
+    def next
+      if @cursor
+        @cursor = @list.next(@cursor)
+      else
+        @cursor = @list.first
+      end
+    end
+  end
+end
+
+class Innodb::List::Xdes < Innodb::List
+  def object_from_address(address)
+    if address && page = @space.page(address[:page])
+      Innodb::Xdes.new(page, page.cursor(address[:offset] - 8))
+    end
+  end
+end
+
+class Innodb::List::Inode < Innodb::List
+  def object_from_address(address)
+    if address && page = @space.page(address[:page])
+      page
+    end
+  end
+end

data/lib/innodb/page.rb

@@ -1,5 +1,10 @@
 require "innodb/cursor"
 
+# A generic class for any type of page, which handles reading the common
+# FIL header and trailer, and can handle (via #parse) dispatching to a more
+# specialized class depending on page type (which comes from the FIL header).
+# A page being handled by Innodb::Page indicates that its type is not currently
+# handled by any more specialized class.
 class Innodb::Page
   SPECIALIZED_CLASSES = {}
 
@@ -7,11 +12,11 @@ class Innodb::Page
   # and then attempt to hand off the page to a specialized class to be
   # re-parsed if possible. If there is no specialized class for this type
   # of page, return the generic object.
-  def self.parse(buffer)
-    page = Innodb::Page.new(buffer)
+  def self.parse(space, buffer)
+    page = Innodb::Page.new(space, buffer)
 
     if specialized_class = SPECIALIZED_CLASSES[page.type]
-      page = specialized_class.new(buffer)
+      page = specialized_class.new(space, buffer)
     end
 
     page
@@ -19,7 +24,8 @@ class Innodb::Page
 
   # Initialize a page by passing in a 16kB buffer containing the raw page
   # contents. Currently only 16kB pages are supported.
-  def initialize(buffer)
+  def initialize(space, buffer)
+    @space  = space
     @buffer = buffer
   end
 
@@ -130,6 +136,22 @@ class Innodb::Page
     fil_header[:lsn]
   end
 
+  def inspect
+    if fil_header
+      "#<%s: size=%i, space_id=%i, offset=%i, type=%s, prev=%i, next=%i>" % [
+        self.class,
+        size,
+        fil_header[:space_id],
+        fil_header[:offset],
+        fil_header[:type],
+        fil_header[:prev],
+        fil_header[:next],
+      ]
+    else
+      "#<#{self.class}>"
+    end
+  end
+
   # Dump the contents of a page for debugging purposes.
   def dump
     puts "#{self}:"

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -1,24 +1,38 @@
-require "innodb/free_list"
+require "innodb/list"
+require "innodb/xdes"
 
+# A specialized class for FSP_HDR (filespace header) and XDES (extent
+# descriptor) page types. Each tablespace always has an FSP_HDR page as
+# its first page (page 0), and has repeating XDES pages every 16,384 pages
+# after that (page 16384, 32768, ...). The FSP_HDR and XDES page structure
+# is completely identical, with the exception that the FSP header structure
+# is zero-filled on XDES pages, but populated on FSP_HDR pages.
+#
+# The basic structure of FSP_HDR and XDES pages is: FIL header, FSP header,
+# an array of 256 XDES entries, empty (unused) space, and FIL trailer.
 class Innodb::Page::FspHdrXdes < Innodb::Page
-  XDES_BITS_PER_PAGE = 2
-  XDES_BITMAP_SIZE = (64 * XDES_BITS_PER_PAGE) / 8
-  XDES_SIZE = 8 + Innodb::FreeList::NODE_SIZE + 4 + XDES_BITMAP_SIZE
-
+  # This is actually defined as page size divided by extent size, which is
+  # 16384 / 64 = 256.
   XDES_N_ARRAY_ENTRIES = 256
 
+  # The FSP header immediately follows the FIL header.
   def pos_fsp_header
     pos_fil_header + size_fil_header
   end
 
+  # The FSP header contains six 32-bit integers, one 64-bit integer, and 5
+  # list base nodes.
   def size_fsp_header
-    (32 + 5 * Innodb::FreeList::BASE_NODE_SIZE)
+    ((4 * 6) + (1 * 8) + (5 * Innodb::List::BASE_NODE_SIZE))
   end
 
+  # The XDES entry array immediately follows the FSP header.
   def pos_xdes_array
     pos_fsp_header + size_fsp_header
   end
 
+  # Read the FSP (filespace) header, which contains a few counters and flags,
+  # as well as list base nodes for each list maintained in the filespace.
   def fsp_header
     c = cursor(pos_fsp_header)
     @fsp_header ||= {
@@ -28,36 +42,28 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
       :free_limit         => c.get_uint32,
       :flags              => c.get_uint32,
       :frag_n_used        => c.get_uint32,
-      :free               => Innodb::FreeList::get_base_node(c),
-      :free_frag          => Innodb::FreeList::get_base_node(c),
-      :full_frag          => Innodb::FreeList::get_base_node(c),
+      :free               => Innodb::List::Xdes.new(@space,
+                              Innodb::List.get_base_node(c)),
+      :free_frag          => Innodb::List::Xdes.new(@space,
+                              Innodb::List.get_base_node(c)),
+      :full_frag          => Innodb::List::Xdes.new(@space,
+                              Innodb::List.get_base_node(c)),
       :first_unused_seg   => c.get_uint64,
-      :full_inodes        => Innodb::FreeList::get_base_node(c),
-      :free_inodes        => Innodb::FreeList::get_base_node(c),
+      :full_inodes        => Innodb::List::Inode.new(@space,
+                              Innodb::List.get_base_node(c)),
+      :free_inodes        => Innodb::List::Inode.new(@space,
+                              Innodb::List.get_base_node(c)),
     }
   end
 
-  XDES_STATES = {
-    1 => :free,
-    2 => :free_frag,
-    3 => :full_frag,
-    4 => :fseg,
-  }
-
-  def read_xdes(cursor)
-    {
-      :xdes_id    => cursor.get_uint64,
-      :position   => cursor.position,
-      :free_list  => Innodb::FreeList::get_node(cursor),
-      :state      => XDES_STATES[cursor.get_uint32],
-      :bitmap     => cursor.get_hex(XDES_BITMAP_SIZE),
-    }
-  end
-  
+  # Iterate through all XDES entries in order. This is useful for debugging,
+  # but each of these entries is actually a node in some other list. The state
+  # field in the XDES entry indicates which type of list it is present in,
+  # although not necessarily which list (e.g. :fseg).
   def each_xdes
     c = cursor(pos_xdes_array)
     XDES_N_ARRAY_ENTRIES.times do
-      yield read_xdes(c)
+      yield Innodb::Xdes.new(self, c)
     end
   end
 

data/lib/innodb/page/index.rb

@@ -1,5 +1,13 @@
 require "innodb/fseg_entry"
 
+# A specialized class for handling INDEX pages, which contain a portion of
+# the data from exactly one B+tree. These are typically the most common type
+# of page in any database.
+#
+# The basic structure of an INDEX page is: FIL header, INDEX header, FSEG
+# header, fixed-width system records (infimum and supremum), user records
+# (the actual data) which grow ascending by offset, free space, the page
+# directory which grows descending by offset, and the FIL trailer.
 class Innodb::Page::Index < Innodb::Page
   attr_accessor :record_describer
 
@@ -158,16 +166,26 @@ class Innodb::Page::Index < Innodb::Page
   def fseg_header
     c = cursor(pos_fseg_header)
     @fseg_header ||= {
-      :free_list      => Innodb::FsegEntry.get_entry(c),
-      :btree_segment  => Innodb::FsegEntry.get_entry(c),
+      :leaf     => Innodb::FsegEntry.get_inode(@space, c),
+      :internal => Innodb::FsegEntry.get_inode(@space, c),
     }
   end
 
+  # The size (in bytes) of the bit-packed fields in each record header.
   RECORD_BITS_SIZE  = 3
+
+  # The size (in bytes) of the "next" pointer in each record header.
   RECORD_NEXT_SIZE  = 2
 
+  # The size (in bytes) of the record pointers in each page directory slot.
   PAGE_DIR_SLOT_SIZE          = 2
+
+  # The minimum number of records "owned" by each record with an entry in
+  # the page directory.
   PAGE_DIR_SLOT_MIN_N_OWNED   = 4
+
+  # The maximum number of records "owned" by each record with an entry in
+  # the page directory.
   PAGE_DIR_SLOT_MAX_N_OWNED   = 8
 
   # Return the size of the header for each record.
@@ -205,7 +223,7 @@ class Innodb::Page::Index < Innodb::Page
   # This record has been marked as deleted.
   RECORD_INFO_DELETED_FLAG = 2
 
-  # Return the header from a record. (This is mostly unimplemented.)
+  # Return the header from a record.
   def record_header(offset)
     return nil unless type == :INDEX
 
@@ -222,12 +240,70 @@ class Innodb::Page::Index < Innodb::Page
       info = (bits2 & 0xf0) >> 4
       header[:min_rec] = (info & RECORD_INFO_MIN_REC_FLAG) != 0
       header[:deleted] = (info & RECORD_INFO_DELETED_FLAG) != 0
+      case header[:type]
+      when :conventional, :node_pointer:
+        # The variable-length part of the record header contains a
+        # bit vector indicating NULL fields and the length of each
+        # non-NULL variable-length field.
+        if record_format
+          header[:null_bitmap] = nbmap = record_null_bitmap(c)
+          header[:variable_length] = record_variable_length(c, nbmap)
+        end
+      end
       header
     when :redundant
       raise "Not implemented"
     end
   end
 
+  # Return an array indicating which fields are null.
+  def record_null_bitmap(cursor)
+    fields = (record_format[:key] + record_format[:row])
+
+    # The number of bits in the bitmap is the number of nullable fields.
+    size = fields.count do |f| f.nullable end
+
+    # There is no bitmap if there are no nullable fields.
+    return nil unless size > 0
+
+    # To simplify later checks, expand bitmap to one for each field.
+    bitmap = Array.new(fields.size, false)
+
+    null_bit_array = cursor.get_bit_array(size).reverse!
+
+    # For every nullable field, set whether the field is actually null.
+    fields.each do |f|
+      bitmap[f.position] = f.nullable ? (null_bit_array.shift == 1) : false
+    end
+
+    return bitmap
+  end
+
+  # Return an array containing the length of each variable-length field.
+  def record_variable_length(cursor, null_bitmap)
+    fields = (record_format[:key] + record_format[:row])
+
+    len_array = Array.new(fields.size, 0)
+
+    # For each non-NULL variable-length field, the record header contains
+    # the length in one or two bytes.
+    fields.each do |f|
+      next if f.fixed_len > 0 or null_bitmap[f.position]
+
+      len = cursor.get_uint8
+
+      # Two bytes are used only if the length exceeds 127 bytes and the
+      # maximum length exceeds 255 bytes.
+      if len > 127 and f.variable_len > 255
+        len = ((len & 0x3f) << 8) + cursor.get_uint8
+      end
+
+      len_array[f.position] = len
+    end
+
+    return len_array
+  end
+
   # Parse and return simple fixed-format system records, such as InnoDB's
   # internal infimum and supremum records.
   def system_record(offset)
@@ -235,6 +311,7 @@ class Innodb::Page::Index < Innodb::Page
 
     header = record_header(offset)
     {
+      :offset => offset,
       :header => header,
       :next => offset + header[:next],
       :data => cursor(offset).get_bytes(size_mum_record),
@@ -251,10 +328,28 @@ class Innodb::Page::Index < Innodb::Page
     @supremum ||= system_record(pos_supremum)
   end
 
+  # Return a set of field objects that describe the record.
+  def make_record_description
+    description = record_describer.cursor_sendable_description(self)
+
+    fields = []
+
+    (description[:key] + description[:row]).each_with_index do |d, p|
+      fields << Innodb::Field.new(p, *d)
+    end
+
+    n = description[:key].size
+
+    description[:key] = fields.slice(0 .. n-1)
+    description[:row] = fields.slice(n ..  -1)
+
+    return description
+  end
+
   # Return (and cache) the record format provided by an external class.
   def record_format
     if record_describer
-      @record_format ||= record_describer.cursor_sendable_description(self)
+      @record_format ||= make_record_description()
     end
   end
 
@@ -271,6 +366,7 @@ class Innodb::Page::Index < Innodb::Page
     header = record_header(offset)
 
     this_record = {
+      :format => page_header[:format],
       :offset => offset,
       :header => header,
       :next => header[:next] == 0 ? nil : (offset + header[:next]),
@@ -282,7 +378,7 @@ class Innodb::Page::Index < Innodb::Page
       # Read the key fields present in all types of pages.
       this_record[:key] = []
       record_format[:key].each do |f|
-        this_record[:key].push c.send(*f)
+        this_record[:key].push f.read(this_record, c)
       end
 
       # If this is a leaf page of the clustered index, read InnoDB's internal
@@ -299,7 +395,7 @@ class Innodb::Page::Index < Innodb::Page
         # Read the non-key fields.
         this_record[:row] = []
         record_format[:row].each do |f|
-          this_record[:row].push c.send(*f)
+          this_record[:row].push f.read(this_record, c)
         end
       end
 
@@ -427,4 +523,4 @@ class Innodb::Page::Index < Innodb::Page
   end
 end
 
-Innodb::Page::SPECIALIZED_CLASSES[:INDEX] = Innodb::Page::Index
+Innodb::Page::SPECIALIZED_CLASSES[:INDEX] = Innodb::Page::Index