innodb_ruby 0.8.8 → 0.9.0

data/lib/innodb/index.rb

@@ -3,12 +3,9 @@
 # An InnoDB index B-tree, given an Innodb::Space and a root page number.
 class Innodb::Index
   attr_reader :root
-  attr_reader :stats
-  attr_accessor :debug
   attr_accessor :record_describer
 
   def initialize(space, root_page_number, record_describer=nil)
-    @debug = false
     @space = space
     @record_describer = record_describer || space.record_describer
 
@@ -27,8 +24,6 @@ class Innodb::Index
     unless @root.prev.nil? && @root.next.nil?
       raise "Page #{root_page_number} is a node page, but not appear to be the root; it has previous page and next page pointers"
     end
-
-    reset_stats
   end
 
   def page(page_number)
@@ -37,10 +32,6 @@ class Innodb::Index
     page
   end
 
-  def reset_stats
-    @stats = Hash.new(0)
-  end
-
   # A helper function to access the index ID in the page header.
   def id
     @root.page_header[:index_id]
@@ -65,7 +56,7 @@ class Innodb::Index
 
     parent_page.each_child_page do |child_page_number, child_min_key|
       child_page = page(child_page_number)
-      child_page.record_describer = @space.record_describer
+      child_page.record_describer = record_describer
       if child_page.type == :INDEX
         if link_proc
           link_proc.call(parent_page, child_page, child_min_key, depth+1)
@@ -83,16 +74,42 @@ class Innodb::Index
 
   # Return the first leaf page in the index by walking down the left side
   # of the B-tree until a page at the given level is encountered.
-  def first_page_at_level(level)
+  def min_page_at_level(level)
+    page = @root
+    record = @root.min_record
+    while record && page.level > level
+      page = page(record.child_page_number)
+      record = page.min_record
+    end
+    page if page.level == level
+  end
+
+  # Return the minimum record in the index.
+  def min_record
+    if min_page = min_page_at_level(0)
+      min_page.min_record
+    end
+  end
+
+  # Return the last leaf page in the index by walking down the right side
+  # of the B-tree until a page at the given level is encountered.
+  def max_page_at_level(level)
     page = @root
-    record = @root.first_record
+    record = @root.max_record
     while record && page.level > level
       page = page(record.child_page_number)
-      record = page.first_record
+      record = page.max_record
     end
     page if page.level == level
   end
 
+  # Return the maximum record in the index.
+  def max_record
+    if max_page = max_page_at_level(0)
+      max_page.max_record
+    end
+  end
+
   # Return the file segment with the given name from the fseg header.
   def fseg(name)
     @root.fseg_header[name]
@@ -155,7 +172,7 @@ class Innodb::Index
       return enum_for(:each_page_at_level, level)
     end
 
-    each_page_from(first_page_at_level(level)) { |page| yield page }
+    each_page_from(min_page_at_level(level)) { |page| yield page }
   end
 
   # Iterate through all records on all leaf pages in ascending order.
@@ -171,173 +188,17 @@ class Innodb::Index
     end
   end
 
-  # Compare two arrays of fields to determine if they are equal. This follows
-  # the same comparison rules as strcmp and others:
-  #   0 = a is equal to b
-  #   -1 = a is less than b
-  #   +1 = a is greater than b
-  def compare_key(a, b)
-    @stats[:compare_key] += 1
-
-    return 0 if a.nil? && b.nil?
-    return -1 if a.nil? || (!b.nil? && a.size < b.size)
-    return +1 if b.nil? || (!a.nil? && a.size > b.size)
-
-    a.each_index do |i|
-      @stats[:compare_key_field_comparison] += 1
-      return -1 if a[i] < b[i][:value]
-      return +1 if a[i] > b[i][:value]
-    end
-
-    return 0
-  end
-
-  # Search for a record within a single page, and return either a perfect
-  # match for the key, or the last record closest to they key but not greater
-  # than the key. (If an exact match is desired, compare_key must be used to
-  # check if the returned record matches. This makes the function useful for
-  # search in both leaf and non-leaf pages.)
-  def linear_search_from_cursor(page, cursor, key)
-    @stats[:linear_search_from_cursor] += 1
-
-    this_rec = cursor.record
-
-    if @debug
-      puts "linear_search_from_cursor: page=%i, level=%i, start=(%s)" % [
-        page.offset,
-        page.level,
-        this_rec && this_rec.key_string,
-      ]
-    end
-
-    # Iterate through all records until finding either a matching record or
-    # one whose key is greater than the desired key.
-    while this_rec && next_rec = cursor.record
-      @stats[:linear_search_from_cursor_record_scans] += 1
-
-      if @debug
-        puts "linear_search_from_cursor: page=%i, level=%i, current=(%s)" % [
-          page.offset,
-          page.level,
-          this_rec && this_rec.key_string,
-        ]
-      end
-
-      # If we reach supremum, return the last non-system record we got.
-      return this_rec if next_rec.header[:type] == :supremum
-
-      if compare_key(key, this_rec.key) < 0
-        return this_rec
-      end
-
-      if (compare_key(key, this_rec.key) >= 0) &&
-        (compare_key(key, next_rec.key) < 0)
-        # The desired key is either an exact match for this_rec or is greater
-        # than it but less than next_rec. If this is a non-leaf page, that
-        # will mean that the record will fall on the leaf page this node
-        # pointer record points to, if it exists at all.
-        return this_rec
-      end
-
-      this_rec = next_rec
-    end
-
-    this_rec
-  end
-
-  # Search or a record within a single page using the page directory to limit
-  # the number of record comparisons required. Once the last page directory
-  # entry closest to but not greater than the key is found, fall back to
-  # linear search using linear_search_from_cursor to find the closest record
-  # whose key is not greater than the desired key. (If an exact match is
-  # desired, the returned record must be checked in the same way as the above
-  # linear_search_from_cursor function.)
-  def binary_search_by_directory(page, dir, key)
-    @stats[:binary_search_by_directory] += 1
-
-    return nil if dir.empty?
-
-    # Split the directory at the mid-point (using integer math, so the division
-    # is rounding down). Retrieve the record that sits at the mid-point.
-    mid = ((dir.size-1) / 2)
-    rec = page.record(dir[mid])
-
-    if @debug
-      puts "binary_search_by_directory: page=%i, level=%i, dir.size=%i, dir[%i]=(%s)" % [
-        page.offset,
-        page.level,
-        dir.size,
-        mid,
-        rec.key_string,
-      ]
-    end
-
-    # The mid-point record was the infimum record, which is not comparable with
-    # compare_key, so we need to just linear scan from here. If the mid-point
-    # is the beginning of the page there can't be many records left to check
-    # anyway.
-    if rec.header[:type] == :infimum
-      return linear_search_from_cursor(page, page.record_cursor(rec.next), key)
-    end
-
-    # Compare the desired key to the mid-point record's key.
-    case compare_key(key, rec.key)
-    when 0
-      # An exact match for the key was found. Return the record.
-      @stats[:binary_search_by_directory_exact_match] += 1
-      rec
-    when +1
-      # The mid-point record's key is less than the desired key.
-      if dir.size > 2
-        # There are more entries remaining from the directory, recurse again
-        # using binary search on the right half of the directory, which
-        # represents values greater than or equal to the mid-point record's
-        # key.
-        @stats[:binary_search_by_directory_recurse_right] += 1
-        binary_search_by_directory(page, dir[mid...dir.size], key)
-      else
-        next_rec = page.record(dir[mid+1])
-        next_key = next_rec && compare_key(key, next_rec.key)
-        if dir.size == 1 || next_key == -1 || next_key == 0
-          # This is the last entry remaining from the directory, or our key is
-          # greater than rec and less than rec+1's key. Use linear search to
-          # find the record starting at rec.
-          @stats[:binary_search_by_directory_linear_search] += 1
-          linear_search_from_cursor(page, page.record_cursor(rec.offset), key)
-        elsif next_key == +1
-          @stats[:binary_search_by_directory_linear_search] += 1
-          linear_search_from_cursor(page, page.record_cursor(next_rec.offset), key)
-        else
-          nil
-        end
-      end
-    when -1
-      # The mid-point record's key is greater than the desired key.
-      if dir.size == 1
-        # If this is the last entry remaining from the directory, we didn't
-        # find anything workable.
-        @stats[:binary_search_by_directory_empty_result] += 1
-        nil
-      else
-        # Recurse on the left half of the directory, which represents values
-        # less than the mid-point record's key.
-        @stats[:binary_search_by_directory_recurse_left] += 1
-        binary_search_by_directory(page, dir[0...mid], key)
-      end
-    end
-  end
-
   # Search for a record within the entire index, walking down the non-leaf
   # pages until a leaf page is found, and then verifying that the record
   # returned on the leaf page is an exact match for the key. If a matching
   # record is not found, nil is returned (either because linear_search_in_page
   # returns nil breaking the loop, or because compare_key returns non-zero).
   def linear_search(key)
-    @stats[:linear_search] += 1
+    Innodb::Stats.increment :linear_search
 
     page = @root
 
-    if @debug
+    if Innodb.debug?
       puts "linear_search: root=%i, level=%i, key=(%s)" % [
         page.offset,
         page.level,
@@ -346,7 +207,7 @@ class Innodb::Index
     end
 
     while rec =
-      linear_search_from_cursor(page, page.record_cursor(page.infimum.next), key)
+      page.linear_search_from_cursor(page.record_cursor(page.infimum.next), key)
       if page.level > 0
         # If we haven't reached a leaf page yet, move down the tree and search
         # again using linear search.
@@ -355,7 +216,7 @@ class Innodb::Index
         # We're on a leaf page, so return the page and record if there is a
         # match. If there is no match, break the loop and cause nil to be
         # returned.
-        return page, rec if compare_key(key, rec.key) == 0
+        return rec if rec.compare_key(key) == 0
         break
       end
     end
@@ -365,11 +226,11 @@ class Innodb::Index
   # the page directory to search while making as few record comparisons as
   # possible. If a matching record is not found, nil is returned.
   def binary_search(key)
-    @stats[:binary_search] += 1
+    Innodb::Stats.increment :binary_search
 
     page = @root
 
-    if @debug
+    if Innodb.debug?
       puts "binary_search: root=%i, level=%i, key=(%s)" % [
         page.offset,
         page.level,
@@ -379,7 +240,7 @@ class Innodb::Index
 
     # Remove supremum from the page directory, since nothing can be scanned
     # linearly from there anyway.
-    while rec = binary_search_by_directory(page, page.directory[0...-1], key)
+    while rec = page.binary_search_by_directory(page.directory[0...-1], key)
       if page.level > 0
         # If we haven't reached a leaf page yet, move down the tree and search
         # again using binary search.
@@ -388,10 +249,120 @@ class Innodb::Index
         # We're on a leaf page, so return the page and record if there is a
         # match. If there is no match, break the loop and cause nil to be
         # returned.
-        return page, rec if compare_key(key, rec.key) == 0
+        return rec if rec.compare_key(key) == 0
         break
       end
     end
   end
 
+  # A cursor to walk the index (cursor) forwards or backward starting with
+  # a given record, or the minimum (:min) or maximum (:max) record in the
+  # index.
+  class IndexCursor
+    def initialize(index, record, direction)
+      Innodb::Stats.increment :index_cursor_create
+      @initial = true
+      @index = index
+      @direction = direction
+      case record
+      when :min
+        # Start at the minimum record on the minimum page in the index.
+        @page = index.min_page_at_level(0)
+        @page_cursor = @page.record_cursor(:min, direction)
+      when :max
+        # Start at the maximum record on the maximum page in the index.
+        @page = index.max_page_at_level(0)
+        @page_cursor = @page.record_cursor(:max, direction)
+      else
+        # Start at the record provided.
+        @page = record.page
+        @page_cursor = @page.record_cursor(record.offset, direction)
+      end
+      @record = @page_cursor.record
+    end
+
+    # Return the current record, mostly as a helper.
+    def current_record
+      @record
+    end
+
+    # Move to the next record in the forward direction and return it.
+    def next_record
+      Innodb::Stats.increment :index_cursor_next_record
+
+      while true
+        if rec = @page_cursor.record
+          return rec
+        end
+
+        unless next_page = @page.next
+          return nil
+        end
+
+        unless @page = @index.page(next_page)
+          raise "Failed to load next page"
+        end
+
+        unless @page_cursor = @page.record_cursor(:min, @direction)
+          raise "Failed to position cursor"
+        end
+      end
+    end
+
+    # Move to the previous record in the backward direction and return it.
+    def prev_record
+      Innodb::Stats.increment :index_cursor_prev_record
+
+      while true
+        if rec = @page_cursor.record
+          return rec
+        end
+
+        unless prev_page = @page.prev
+          return nil
+        end
+
+        unless @page = @index.page(prev_page)
+          raise "Failed to load prev page"
+        end
+
+        unless @page_cursor = @page.record_cursor(:max, @direction)
+          raise "Failed to position cursor"
+        end
+      end
+      raise "Not implemented"
+    end
+
+    # Return the next record in the order defined when the cursor was created.
+    def record
+      if @initial
+        @initial = false
+        return current_record
+      end
+
+      case @direction
+      when :forward
+        next_record
+      when :backward
+        prev_record
+      end
+    end
+
+    # Iterate through all records in the cursor.
+    def each_record
+      unless block_given?
+        return enum_for(:each_record)
+      end
+
+      while rec = record
+        yield rec
+      end
+    end
+  end
+
+  # Return an IndexCursor starting at the given record (an Innodb::Record,
+  # :min, or :max) and cursor in the direction given (:forward or :backward).
+  def cursor(record=:min, direction=:forward)
+    IndexCursor.new(self, record, direction)
+  end
 end

data/lib/innodb/log.rb

@@ -3,50 +3,113 @@
 # An InnoDB transaction log file.
 
 class Innodb::Log
-  HEADER_SIZE   = 4 * Innodb::LogBlock::BLOCK_SIZE
-  HEADER_START  = 0
-  DATA_START    = HEADER_START + HEADER_SIZE
-
-#define LOG_GROUP_ID		0	/* log group number */
-#define LOG_FILE_START_LSN	4	/* lsn of the start of data in this
-#define LOG_FILE_NO		12	/* 4-byte archived log file number;
-#define LOG_FILE_WAS_CREATED_BY_HOT_BACKUP 16
-#define	LOG_FILE_ARCH_COMPLETED	OS_FILE_LOG_BLOCK_SIZE
-#define LOG_FILE_END_LSN	(OS_FILE_LOG_BLOCK_SIZE + 4)
-#define LOG_CHECKPOINT_1	OS_FILE_LOG_BLOCK_SIZE
-#define LOG_CHECKPOINT_2	(3 * OS_FILE_LOG_BLOCK_SIZE)
-#define LOG_FILE_HDR_SIZE	(4 * OS_FILE_LOG_BLOCK_SIZE)
+  # A map of the name and position of the blocks that form the log header.
+  LOG_HEADER_BLOCK_MAP = {
+    :LOG_FILE_HEADER  => 0,
+    :LOG_CHECKPOINT_1 => 1,
+    :EMPTY            => 2,
+    :LOG_CHECKPOINT_2 => 3,
+  }
+
+  # Number of blocks in the log file header.
+  LOG_HEADER_BLOCKS = LOG_HEADER_BLOCK_MAP.size
+
+  # Maximum number of log group checkpoints.
+  LOG_CHECKPOINT_GROUPS = 32
 
   # Open a log file.
   def initialize(file)
-    @name = file
-    File.open(@name) do |log|
-      @size = log.stat.size
-      @blocks = ((@size - DATA_START) / Innodb::LogBlock::BLOCK_SIZE)
-    end
+    @file = File.open(file)
+    @size = @file.stat.size
+    @blocks = (@size / Innodb::LogBlock::BLOCK_SIZE) - LOG_HEADER_BLOCKS
   end
-  
+
+  # The size (in bytes) of the log.
+  attr_reader :size
+
+  # The number of blocks in the the log.
   attr_reader :blocks
 
-  # Return a log block with a given block number as an InnoDB::LogBlock object.
-  # Blocks are numbered after the log file header, starting from 0.
-  def block(block_number)
-    offset = DATA_START + (block_number.to_i * Innodb::LogBlock::BLOCK_SIZE)
-    return nil unless offset < @size
-    return nil unless (offset + Innodb::LogBlock::BLOCK_SIZE) <= @size
-    File.open(@name) do |log|
-      log.seek(offset)
-      block_data = log.read(Innodb::LogBlock::BLOCK_SIZE)
-      Innodb::LogBlock.new(block_data)
+  # Get the raw byte buffer for a specific block by block offset.
+  def block_data(offset)
+    raise "Invalid block offset" unless (offset % Innodb::LogBlock::BLOCK_SIZE).zero?
+    @file.seek(offset)
+    @file.read(Innodb::LogBlock::BLOCK_SIZE)
+  end
+
+  # Get a cursor to a block in a given offset of the log.
+  def block_cursor(offset)
+    Innodb::Cursor.new(block_data(offset), 0)
+  end
+
+  # Return the log header.
+  def header
+    offset = LOG_HEADER_BLOCK_MAP[:LOG_FILE_HEADER] * Innodb::LogBlock::BLOCK_SIZE
+    @header ||= block_cursor(offset).name("header") do |c|
+      {
+        :group_id   => c.name("group_id")   { c.get_uint32 },
+        :start_lsn  => c.name("start_lsn")  { c.get_uint64 },
+        :created_by => c.name("created_by") { c.seek(16).get_bytes(4) }
+      }
     end
   end
 
-  # Iterate through all log blocks, returning the block number and an
+  # Read a log checkpoint from the given cursor.
+  def read_checkpoint(c)
+    # Log archive related fields (e.g. group_array) are not currently in
+    # use or even read by InnoDB. However, for the sake of completeness,
+    # they are included.
+    {
+      :number         => c.name("number")       { c.get_uint64 },
+      :lsn            => c.name("lsn")          { c.get_uint64 },
+      :offset         => c.name("offset")       { c.get_uint32 },
+      :buffer_size    => c.name("buffer_size")  { c.get_uint32 },
+      :archived_lsn   => c.name("archived_lsn") { c.get_uint64 },
+      :group_array    =>
+        (0 .. LOG_CHECKPOINT_GROUPS - 1).map do |n|
+          c.name("group_array[#{n}]") do
+            {
+              :archived_file_no => c.name("archived_file_no") { c.get_uint32 },
+              :archived_offset  => c.name("archived_offset")  { c.get_uint32 },
+            }
+          end
+        end,
+      :checksum_1     => c.name("checksum_1")     { c.get_uint32 },
+      :checksum_2     => c.name("checksum_2")     { c.get_uint32 },
+      :fsp_free_limit => c.name("fsp_free_limit") { c.get_uint32 },
+      :fsp_magic      => c.name("fsp_magic")      { c.get_uint32 },
+    }
+  end
+
+  # Return the log checkpoints.
+  def checkpoint
+    offset1 = LOG_HEADER_BLOCK_MAP[:LOG_CHECKPOINT_1] * Innodb::LogBlock::BLOCK_SIZE
+    offset2 = LOG_HEADER_BLOCK_MAP[:LOG_CHECKPOINT_2] * Innodb::LogBlock::BLOCK_SIZE
+    @checkpoint ||=
+      {
+        :checkpoint_1 => block_cursor(offset1).name("checkpoint_1") do |cursor|
+          read_checkpoint(cursor)
+        end,
+        :checkpoint_2 => block_cursor(offset2).name("checkpoint_2") do |cursor|
+          read_checkpoint(cursor)
+        end
+      }
+  end
+
+  # Return a log block with a given block index as an InnoDB::LogBlock object.
+  # Blocks are indexed after the log file header, starting from 0.
+  def block(block_index)
+    return nil unless block_index.between?(0, @blocks - 1)
+    offset = (LOG_HEADER_BLOCKS + block_index.to_i) * Innodb::LogBlock::BLOCK_SIZE
+    Innodb::LogBlock.new(block_data(offset))
+  end
+
+  # Iterate through all log blocks, returning the block index and an
   # InnoDB::LogBlock object for each block.
   def each_block
-    (0...@blocks).each do |block_number|
-      current_block = block(block_number)
-      yield block_number, current_block if current_block
+    (0...@blocks).each do |block_index|
+      current_block = block(block_index)
+      yield block_index, current_block if current_block
     end
   end
 end