innodb_ruby 0.7.1 → 0.7.2

data/bin/innodb_space

@@ -141,11 +141,10 @@ def print_xdes_list(list)
   list.each do |entry|
     puts "%-12i%-64s" % [
       entry.xdes[:start_page],
-      entry.xdes[:bitmap].bytes.map { |byte|
-        [0, 2, 4, 6].map { |shift|
-          ((byte >> shift) & 1 == 1) ? "." : "#"
-        }
-      }.flatten.join,
+      entry.each_page_status.inject("") { |bitmap, page_status|
+        bitmap += page_status[:free] ? "." : "#"
+        bitmap
+      },
     ]
   end
 end

data/lib/innodb/fseg_entry.rb

@@ -1,6 +1,8 @@
 # An InnoDB file segment entry, which appears in a few places, such as the
 # FSEG header of INDEX pages, and in the TRX_SYS pages.
 class Innodb::FsegEntry
+  # The size (in bytes) of an FSEG entry, which contains a two 32-bit integers
+  # and a 16-bit integer.
   SIZE = 4 + 4 + 2
 
   # Return the FSEG entry address, which points to an entry on an INODE page.

data/lib/innodb/index.rb

@@ -75,6 +75,10 @@ class Innodb::Index
 
   # Iterate through all pages at this level starting with the provided page.
   def each_page_from(page)
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_page_from, page)
+    end
+
     while page && page.type == :INDEX
       yield page
       page = @space.page(page.next)
@@ -84,11 +88,19 @@ class Innodb::Index
   # Iterate through all pages at the given level by finding the first page
   # and following the next pointers in each page.
   def each_page_at_level(level)
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_page_at_level, level)
+    end
+
     each_page_from(first_page_at_level(level)) { |page| yield page }
   end
 
   # Iterate through all records on all leaf pages in ascending order.
   def each_record
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_record)
+    end
+
     each_page_at_level(0) do |page|
       page.each_record do |record|
         yield record
@@ -230,7 +242,7 @@ class Innodb::Index
   def binary_search(key)
     page = @root
 
-    while rec = binary_search_by_directory(page, page.directory.dup, key)
+    while rec = binary_search_by_directory(page, page.directory, key)
       if page.level > 0
         # If we haven't reached a leaf page yet, move down the tree and search
         # again using binary search.

data/lib/innodb/list.rb

@@ -1,8 +1,15 @@
+# An abstract InnoDB "free list" or FLST (renamed to just "list" here as it
+# frequently is used for structures that aren't free lists). This class must
+# be sub-classed to provide an appropriate #object_from_address method.
 class Innodb::List
-  FIL_ADDR_SIZE   = 4 + 2
-  NODE_SIZE       = 2 * FIL_ADDR_SIZE
-  BASE_NODE_SIZE  = 4 + (2 * FIL_ADDR_SIZE)
-
+  # An "address", which consists of a page number and byte offset within the
+  # page. This points to the list "node" pointers (prev and next) of the
+  # node, not necessarily the node object.
+  ADDRESS_SIZE = 4 + 2
+
+  # Read a node address from a cursor. Return nil if the address is an end
+  # or "NULL" pointer (the page number is UINT32_MAX), or the address if
+  # valid.
   def self.get_address(cursor)
     page    = Innodb::Page.maybe_undefined(cursor.get_uint32)
     offset  = cursor.get_uint16
@@ -14,6 +21,13 @@ class Innodb::List
     end
   end
 
+  # A list node consists of two addresses: the "previous" node address, and
+  # the "next" node address.
+  NODE_SIZE = 2 * ADDRESS_SIZE
+
+  # Read a node, consisting of two consecutive addresses (:prev and :next)
+  # from a cursor. Either address may be nil, indicating the end of the
+  # linked list.
   def self.get_node(cursor)
     {
       :prev => get_address(cursor),
@@ -21,6 +35,15 @@ class Innodb::List
     }
   end
 
+  # A list base node consists of a list length followed by two addresses:
+  # the "first" node address, and the "last" node address.
+  BASE_NODE_SIZE = 4 + (2 * ADDRESS_SIZE)
+
+  # Read a base node, consisting of a list length followed by two addresses
+  # (:first and :last) from a cursor. Either address may be nil. An empty list
+  # has a :length of 0 and :first and :last are nil. A list with only a single
+  # item will have a :length of 1 and :first and :last will point to the same
+  # address.
   def self.get_base_node(cursor)
     {
       :length => cursor.get_uint32,
@@ -37,43 +60,80 @@ class Innodb::List
   attr_reader :space
   attr_reader :base
 
+  # Abstract #object_from_address method which must be implemented by
+  # sub-classes in order to return a useful object given an object address.
+  def object_from_address(address)
+    raise "#{self.class} must implement object_from_address"
+  end
+
+  # Return the object pointed to by the "previous" address pointer of the
+  # provided object.
   def prev(object)
+    unless object.respond_to? :prev_address
+      raise "Class #{object.class} does not respond to prev_address"
+    end
+
     object_from_address(object.prev_address)
   end
 
+  # Return the object pointed to by the "next" address pointer of the
+  # provided object.
   def next(object)
+    unless object.respond_to? :next_address
+      raise "Class #{object.class} does not respond to next_address"
+    end
+
     object_from_address(object.next_address)
   end
 
+  # Return the first object in the list using the list base node "first"
+  # address pointer.
   def first
     object_from_address(@base[:first])
   end
 
+  # Return the first object in the list using the list base node "last"
+  # address pointer.
   def last
     object_from_address(@base[:last])
   end
 
+  # Return a list cursor for the list.
   def cursor(node=nil)
     Cursor.new(self, node)
   end
 
+  # Iterate through all nodes in the list.
   def each
+    unless block_given?
+      return Enumerable::Enumerator.new(self)
+    end
+
     c = cursor
     while e = c.next
       yield e
     end
   end
 
+  # A list iteration cursor used primarily by the Innodb::List #cursor method
+  # implicitly. Keeps its own state for iterating through lists efficiently.
   class Cursor
     def initialize(list, node=nil)
       @list   = list
       @cursor = node
     end
 
+    # Reset the list cursor to its default starting state, which will allow
+    # iteration forwards from the first entry (using #next) or backwards
+    # from the last entry (using #prev).
     def reset
       @cursor = nil
     end
 
+    # Return the previous entry from the current position, and advance the
+    # cursor position to the returned entry. If the cursor is currently nil,
+    # return the last entry in the list and adjust the cursor position to
+    # that entry.
     def prev
       if @cursor
         @cursor = @list.prev(@cursor)
@@ -82,6 +142,10 @@ class Innodb::List
       end
     end
 
+    # Return the next entry from the current position, and advance the
+    # cursor position to the returned entry. If the cursor is currently nil,
+    # return the first entry in the list and adjust the cursor position to
+    # that entry.
     def next
       if @cursor
         @cursor = @list.next(@cursor)
@@ -92,6 +156,8 @@ class Innodb::List
   end
 end
 
+# A list of extent descriptor entries. Objects returned by list methods
+# will be Innodb::Xdes objects.
 class Innodb::List::Xdes < Innodb::List
   def object_from_address(address)
     if address && page = @space.page(address[:page])
@@ -100,6 +166,8 @@ class Innodb::List::Xdes < Innodb::List
   end
 end
 
+# A list of Inode pages. Objects returned by list methods will be
+# Innodb::Page::Inode objects.
 class Innodb::List::Inode < Innodb::List
   def object_from_address(address)
     if address && page = @space.page(address[:page])

data/lib/innodb/page.rb

@@ -6,15 +6,25 @@ require "innodb/cursor"
 # A page being handled by Innodb::Page indicates that its type is not currently
 # handled by any more specialized class.
 class Innodb::Page
+  # A hash of page types to specialized classes to handle them. Normally
+  # subclasses will register themselves in this list.
   SPECIALIZED_CLASSES = {}
 
   # Load a page as a generic page in order to make the "fil" header accessible,
   # and then attempt to hand off the page to a specialized class to be
   # re-parsed if possible. If there is no specialized class for this type
   # of page, return the generic object.
+  #
+  # This could be optimized to reach into the page buffer and efficiently
+  # extract the page type in order to avoid throwing away a generic
+  # Innodb::Page object when parsing every specialized page, but this is
+  # a bit cleaner, and we're not particularly performance sensitive.
   def self.parse(space, buffer)
+    # Create a page object as a generic page.
     page = Innodb::Page.new(space, buffer)
 
+    # If there is a specialized class available for this page type, re-create
+    # the page object using that specialized class.
     if specialized_class = SPECIALIZED_CLASSES[page.type]
       page = specialized_class.new(space, buffer)
     end
@@ -53,7 +63,7 @@ class Innodb::Page
 
   # Return the size of the "fil" header, in bytes.
   def size_fil_header
-    38
+    4 + 4 + 4 + 4 + 8 + 2 + 8 + 4
   end
 
   # Return the byte offset of the start of the "fil" trailer, which is at
@@ -64,7 +74,7 @@ class Innodb::Page
 
   # Return the size of the "fil" trailer, in bytes.
   def size_fil_trailer
-    8
+    4 + 4
   end
 
   # InnoDB Page Type constants from include/fil0fil.h.
@@ -136,6 +146,8 @@ class Innodb::Page
     fil_header[:lsn]
   end
 
+  # Implement a custom inspect method to avoid irb printing the contents of
+  # the page buffer, since it's very large and mostly not interesting.
   def inspect
     if fil_header
       "#<%s: size=%i, space_id=%i, offset=%i, type=%s, prev=%i, next=%i>" % [

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -61,12 +61,17 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
   # field in the XDES entry indicates which type of list it is present in,
   # although not necessarily which list (e.g. :fseg).
   def each_xdes
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_xdes)
+    end
+
     c = cursor(pos_xdes_array)
     XDES_N_ARRAY_ENTRIES.times do
       yield Innodb::Xdes.new(self, c)
     end
   end
 
+  # Dump the contents of a page for debugging purposes.
   def dump
     super
 

data/lib/innodb/page/index.rb

@@ -116,13 +116,13 @@ class Innodb::Page::Index < Innodb::Page
     data(pos_user_records, page_header[:heap_top] - pos_user_records)
   end
 
-  # Page direction values possible in the page_header[:direction] field.
+  # Page direction values possible in the page_header's :direction field.
   PAGE_DIRECTION = {
-    1 => :left,
-    2 => :right,
-    3 => :same_rec,
-    4 => :same_page,
-    5 => :no_direction,
+    1 => :left,           # Inserts have been in descending order.
+    2 => :right,          # Inserts have been in ascending order.
+    3 => :same_rec,       # Unused by InnoDB.
+    4 => :same_page,      # Unused by InnoDB.
+    5 => :no_direction,   # Inserts have been in random order.
   }
 
   # Return the "index" header.
@@ -172,21 +172,21 @@ class Innodb::Page::Index < Innodb::Page
   end
 
   # The size (in bytes) of the bit-packed fields in each record header.
-  RECORD_BITS_SIZE  = 3
+  RECORD_BITS_SIZE = 3
 
   # The size (in bytes) of the "next" pointer in each record header.
-  RECORD_NEXT_SIZE  = 2
+  RECORD_NEXT_SIZE = 2
 
   # The size (in bytes) of the record pointers in each page directory slot.
-  PAGE_DIR_SLOT_SIZE          = 2
+  PAGE_DIR_SLOT_SIZE = 2
 
   # The minimum number of records "owned" by each record with an entry in
   # the page directory.
-  PAGE_DIR_SLOT_MIN_N_OWNED   = 4
+  PAGE_DIR_SLOT_MIN_N_OWNED = 4
 
   # The maximum number of records "owned" by each record with an entry in
   # the page directory.
-  PAGE_DIR_SLOT_MAX_N_OWNED   = 8
+  PAGE_DIR_SLOT_MAX_N_OWNED = 8
 
   # Return the size of the header for each record.
   def size_record_header
@@ -211,10 +211,10 @@ class Innodb::Page::Index < Innodb::Page
 
   # Record types used in the :type field of the record header.
   RECORD_TYPES = {
-    0 => :conventional,
-    1 => :node_pointer,
-    2 => :infimum,
-    3 => :supremum,
+    0 => :conventional,   # A normal user record in a leaf page.
+    1 => :node_pointer,   # A node pointer in a non-leaf page.
+    2 => :infimum,        # The system "infimum" record.
+    3 => :supremum,       # The system "supremum" record.
   }
 
   # This record is the minimum record at this level of the B-tree.
@@ -225,8 +225,6 @@ class Innodb::Page::Index < Innodb::Page
 
   # Return the header from a record.
   def record_header(offset)
-    return nil unless type == :INDEX
-
     c = cursor(offset).backward
     case page_header[:format]
     when :compact
@@ -307,8 +305,6 @@ class Innodb::Page::Index < Innodb::Page
   # Parse and return simple fixed-format system records, such as InnoDB's
   # internal infimum and supremum records.
   def system_record(offset)
-    return nil unless type == :INDEX
-
     header = record_header(offset)
     {
       :offset => offset,
@@ -356,7 +352,6 @@ class Innodb::Page::Index < Innodb::Page
   # Parse and return a record at a given offset.
   def record(offset)
     return nil unless offset
-    return nil unless type == :INDEX
     return infimum  if offset == pos_infimum
     return supremum if offset == pos_supremum
 
@@ -446,6 +441,10 @@ class Innodb::Page::Index < Innodb::Page
 
   # Iterate through all records.
   def each_record
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_record)
+    end
+
     c = record_cursor(infimum[:next])
 
     while rec = c.record
@@ -461,6 +460,10 @@ class Innodb::Page::Index < Innodb::Page
   def each_child_page
     return nil if level == 0
 
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_child_page)
+    end
+
     each_record do |rec|
       yield rec[:child_page_number], rec[:key]
     end

data/lib/innodb/page/inode.rb

@@ -15,40 +15,59 @@ class Innodb::Page::Inode < Innodb::Page
   # and populated with valid data.
   MAGIC_N_VALUE	= 97937874
 
-  def pos_inode_list_entry
+  # Return the byte offset of the list node, which immediately follows the
+  # FIL header.
+  def pos_list_entry
     pos_fil_header + size_fil_header
   end
 
-  def pos_inode_list
-    pos_fil_header + size_fil_header + Innodb::List::NODE_SIZE
+  # Return the byte offset of the list node.
+  def size_list_entry
+    Innodb::List::NODE_SIZE
   end
 
+  # Return the byte offset of the Inode array in the page, which immediately
+  # follows the list entry.
+  def pos_inode_array
+    pos_list_entry + size_list_entry
+  end
+
+  # The size (in bytes) of an Inode entry.
   def size_inode
     (16 + (3 * Innodb::List::BASE_NODE_SIZE) +
       (FRAG_ARRAY_N_SLOTS * FRAG_SLOT_SIZE))
   end
 
+  # The number of Inode entries that fit on a page.
   def inodes_per_page
-    (size - pos_inode_list - 10) / size_inode
-  end
-
-  def page_number_array(size, cursor)
-    size.times.map { |n| Innodb::Page.maybe_undefined(cursor.get_uint32) }
+    (size - pos_inode_array - 10) / size_inode
   end
 
+  # Return the list entry.
   def list_entry
     c = cursor(pos_inode_list_entry)
     Innodb::List.get_node(c)
   end
 
+  # Return the "previous" address pointer from the list entry. This is used
+  # by Innodb::List::Inode to iterate through Inode lists.
   def prev_address
     list_entry[:prev]
   end
 
+  # Return the "next" address pointer from the list entry. This is used
+  # by Innodb::List::Inode to iterate through Inode lists.
   def next_address
     list_entry[:next]
   end
 
+  # Read an array of page numbers (32-bit integers, which may be nil) from
+  # the provided cursor.
+  def page_number_array(size, cursor)
+    size.times.map { |n| Innodb::Page.maybe_undefined(cursor.get_uint32) }
+  end
+
+  # Read a single Inode entry from the provided cursor.
   def inode(cursor)
     {
       :fseg_id            => cursor.get_uint64,
@@ -64,18 +83,22 @@ class Innodb::Page::Inode < Innodb::Page
     }
   end
 
+  # Read a single Inode entry from the provided byte offset by creating a
+  # cursor and reading the inode using the inode method.
   def inode_at(offset)
     inode(cursor(offset))
   end
 
+  # Iterate through all Inodes in the inode array.
   def each_inode
-    inode_cursor = cursor(pos_inode_list)
+    inode_cursor = cursor(pos_inode_array)
     inodes_per_page.times do
       this_inode = inode(inode_cursor)
       yield this_inode if this_inode[:fseg_id] != 0
     end
   end
 
+  # Dump the contents of a page for debugging purposes.
   def dump
     super
 

data/lib/innodb/page/trx_sys.rb

@@ -90,6 +90,7 @@ class Innodb::Page::TrxSys < Innodb::Page
     }
   end
 
+  # Dump the contents of a page for debugging purposes.
   def dump
     super
 

data/lib/innodb/record_describer.rb

@@ -1 +1,2 @@
+# An empty class in order to establish the Innodb::RecordDescriber namespace.
 class Innodb::RecordDescriber; end

data/lib/innodb/space.rb

@@ -46,6 +46,10 @@ class Innodb::Space
   # root page. This should work fine for IBD files, but not for ibdata
   # files.
   def each_index
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_index)
+    end
+
     (3...@pages).each do |page_number|
       if page(page_number).root?
         yield index(page_number)
@@ -58,6 +62,10 @@ class Innodb::Space
   # Iterate through all pages in a tablespace, returning the page number
   # and an Innodb::Page object for each one.
   def each_page
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_page)
+    end
+
     (0...@pages).each do |page_number|
       current_page = page(page_number)
       yield page_number, current_page if current_page
@@ -67,6 +75,10 @@ class Innodb::Space
   # Iterate through unique regions in the space by page type. This is useful
   # to achieve an overall view of the space.
   def each_page_type_region
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_page_type_region)
+    end
+
     region = nil
     each_page do |page_number, page|
       if region && region[:type] == page.type

data/lib/innodb/version.rb

@@ -1,3 +1,3 @@
 module Innodb
-  VERSION = "0.7.1"
+  VERSION = "0.7.2"
 end

data/lib/innodb/xdes.rb

@@ -1,14 +1,52 @@
+# An InnoDB "extent descriptor entry" or "+XDES+". These structures are used
+# in the +XDES+ entry array contained in +FSP_HDR+ and +XDES+ pages.
+#
+# Note the distinction between +XDES+ _entries_ and +XDES+ _pages_.
 class Innodb::Xdes
+  # Number of pages contained in an extent. InnoDB extents are normally
+  # 64 pages, or 1MiB in size.
   PAGES_PER_EXTENT = 64
+
+  # Number of bits per page in the +XDES+ entry bitmap field. Currently
+  # +XDES+ entries store two bits per page, with the following meanings:
+  #
+  # * 1 = free (the page is free, or not in use)
+  # * 2 = clean (currently unused, always 1 when initialized)
   BITS_PER_PAGE = 2
+
+  # The bit value for a free page.
+  BITMAP_BV_FREE  = 1
+
+  # The bit value for a clean page (currently unused in InnoDB).
+  BITMAP_BV_CLEAN = 2
+
+  # The bitwise-OR of all bitmap bit values.
+  BITMAP_BV_ALL = (BITMAP_BV_FREE | BITMAP_BV_CLEAN)
+
+  # Size (in bytes) of the bitmap field in the +XDES+ entry.
   BITMAP_SIZE = (PAGES_PER_EXTENT * BITS_PER_PAGE) / 8
+
+  # Size (in bytes) of the an +XDES+ entry.
   ENTRY_SIZE = 8 + Innodb::List::NODE_SIZE + 4 + BITMAP_SIZE
 
+  # The values used in the +:state+ field indicating what the extent is
+  # used for (or what list it is on).
   STATES = {
-    1 => :free,
-    2 => :free_frag,
-    3 => :full_frag,
-    4 => :fseg,
+    1 => :free,       # The extent is completely empty and unused, and should
+                      # be present on the filespace's FREE list.
+
+    2 => :free_frag,  # Some pages of the extent are used individually, and
+                      # the extent should be present on the filespace's
+                      # FREE_FRAG list.
+
+    3 => :full_frag,  # All pages of the extent are used individually, and
+                      # the extent should be present on the filespace's
+                      # FULL_FRAG list.
+
+    4 => :fseg,       # The extent is wholly allocated to a file segment.
+                      # Additional information about the state of this extent
+                      # can be derived from the its presence on particular
+                      # file segment lists (FULL, NOT_FULL, or FREE).
   }
 
   def initialize(page, cursor)
@@ -25,15 +63,61 @@ class Innodb::Xdes
     }
   end
 
+  # Return the stored extent descriptor entry.
   def xdes
     @xdes
   end
 
+  # Iterate through all pages represented by this extent descriptor,
+  # yielding a page status hash for each page, containing the following
+  # fields:
+  #
+  #   :page   The page number.
+  #   :free   Boolean indicating whether the page is free.
+  #   :clean  Boolean indicating whether the page is clean (currently
+  #           this bit is unused by InnoDB, and always set true).
+  def each_page_status
+    unless block_given?
+      return Enumerable::Enumerator.new(self, :each_page_status)
+    end
+
+    xdes[:bitmap].bytes.each_with_index do |byte, byte_index|
+      (0..3).each_with_index do |page, page_index|
+        page_number = xdes[:start_page] + (byte_index * 4) + page_index
+        page_bits = ((byte >> (page * BITS_PER_PAGE)) & BITMAP_BV_ALL)
+        page_status = {
+          :page   => page_number,
+          :free   => (page_bits & BITMAP_BV_FREE  != 0),
+          :clean  => (page_bits & BITMAP_BV_CLEAN != 0),
+        }
+        yield page_status
+      end
+    end
+
+    nil
+  end
+
+  # Return the count of free pages (free bit is true) on this extent.
+  def free_pages
+    each_page_status.inject(0) { |sum, p| sum += 1 if p[:free]; sum }
+  end
+
+  # Return the count of used pages (free bit is false) on this extent.
+  def used_pages
+    PAGES_PER_EXTENT - free_pages
+  end
+
+  # Return the address of the previous list pointer from the list node
+  # contained within the XDES entry. This is used by +Innodb::List::Xdes+
+  # to iterate through XDES entries in a list.
   def prev_address
-    @xdes[:list][:prev]
+    xdes[:list][:prev]
   end
 
+  # Return the address of the next list pointer from the list node
+  # contained within the XDES entry. This is used by +Innodb::List::Xdes+
+  # to iterate through XDES entries in a list.
   def next_address
-    @xdes[:list][:next]
+    xdes[:list][:next]
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: innodb_ruby
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 7
   prerelease: 
   segments: 
   - 0
   - 7
-  - 1
-  version: 0.7.1
+  - 2
+  version: 0.7.2
 platform: ruby
 authors: 
 - Jeremy Cole