innodb_ruby 0.8.8 → 0.9.0

data/lib/innodb/log_block.rb

@@ -8,30 +8,14 @@ class Innodb::LogBlock
   # Log blocks are fixed-length at 512 bytes in InnoDB.
   BLOCK_SIZE = 512
 
-  HEADER_SIZE = 12
-  HEADER_START = 0
+  # Offset of the header within the log block.
+  HEADER_OFFSET = 0
 
-  TRAILER_SIZE = 4
-  TRAILER_START = BLOCK_SIZE - TRAILER_SIZE
+  # Offset of the trailer within ths log block.
+  TRAILER_OFFSET = BLOCK_SIZE - 4
 
-  RECORD_START = HEADER_START + HEADER_SIZE
-
-# Header:
-#define	LOG_BLOCK_HDR_NO	0	/* block number which must be > 0 and
-#define	LOG_BLOCK_HDR_DATA_LEN	4	/* number of bytes of log written to
-#define	LOG_BLOCK_FIRST_REC_GROUP 6	/* offset of the first start of an
-#define LOG_BLOCK_CHECKPOINT_NO	8	/* 4 lower bytes of the value of
-
-# Trailer:
-#define	LOG_BLOCK_CHECKSUM	0	/* 4 byte checksum of the log block
-
-#/* Offsets for a checkpoint field */
-#define LOG_CHECKPOINT_NO		0
-#define LOG_CHECKPOINT_LSN		8
-#define LOG_CHECKPOINT_OFFSET		16
-#define LOG_CHECKPOINT_LOG_BUF_SIZE	20
-#define	LOG_CHECKPOINT_ARCHIVED_LSN	24
-#define	LOG_CHECKPOINT_GROUP_ARRAY	32
+  # Mask used to get the flush bit in the header.
+  HEADER_FLUSH_BIT_MASK = 0x80000000
 
   # Initialize a log block by passing in a 512-byte buffer containing the raw
   # log block contents.
@@ -43,36 +27,33 @@ class Innodb::LogBlock
     @buffer = buffer
   end
 
-  # A helper function to return bytes from the log block buffer based on offset
-  # and length, both in bytes.
-  def data(offset, length)
-    @buffer[offset...(offset + length)]
-  end
-
   # Return an Innodb::Cursor object positioned at a specific offset.
   def cursor(offset)
-    Innodb::Cursor.new(self, offset)
+    Innodb::Cursor.new(@buffer, offset)
   end
 
   # Return the log block header.
   def header
-    @header ||= begin
-      c = cursor(HEADER_START)
+    @header ||= cursor(HEADER_OFFSET).name("header") do |c|
       {
-        :block            => c.get_uint32 & (2**31-1),
-        :data_length      => c.get_uint16,
-        :first_rec_group  => c.get_uint16,
-        :checkpoint_no    => c.get_uint32,
+        :flush => c.name("flush") {
+          c.peek { (c.get_uint32 & HEADER_FLUSH_BIT_MASK) > 0 }
+        },
+        :block_number => c.name("block_number") {
+          c.get_uint32 & ~HEADER_FLUSH_BIT_MASK
+        },
+        :data_length      => c.name("data_length")     { c.get_uint16 },
+        :first_rec_group  => c.name("first_rec_group") { c.get_uint16 },
+        :checkpoint_no    => c.name("checkpoint_no")   { c.get_uint32 },
       }
     end
   end
 
   # Return the log block trailer.
   def trailer
-    @trailer ||= begin
-      c = cursor(TRAILER_START)
+    @trailer ||= cursor(TRAILER_OFFSET).name("trailer") do |c|
       {
-        :checksum => c.get_uint32,
+        :checksum => c.name("checksum") { c.get_uint32 },
       }
     end
   end
@@ -127,41 +108,26 @@ class Innodb::LogBlock
     51 => :ZIP_PAGE_COMPRESS,
   }
 
-  # Return the log record contents. (This is mostly unimplemented.)
-  def record_content(record_type, offset)
-    c = cursor(offset)
-    case record_type
-    when :MLOG_1BYTE
-      c.get_uint8
-    when :MLOG_2BYTE
-      c.get_uint16
-    when :MLOG_4BYTE
-      c.get_uint32
-    when :MLOG_8BYTE
-      c.get_uint64
-    when :UNDO_INSERT
-    when :COMP_REC_INSERT
-    end
-  end
-
   SINGLE_RECORD_MASK = 0x80
   RECORD_TYPE_MASK   = 0x7f
 
-  # Return the log record. (This is mostly unimplemented.)
-  def record
-    @record ||= begin
-      if header[:first_rec_group] != 0
-        c = cursor(header[:first_rec_group])
-        type_and_flag = c.get_uint8
-        type = type_and_flag & RECORD_TYPE_MASK
-        type = RECORD_TYPES[type] || type
-        single_record = (type_and_flag & SINGLE_RECORD_MASK) > 0
+  # Return a preamble of the first record in this block.
+  def first_record_preamble
+    return nil unless header[:first_rec_group] > 0
+    cursor(header[:first_rec_group]).name("header") do |c|
+      type_and_flag = c.name("type") { c.get_uint8 }
+      type = type_and_flag & RECORD_TYPE_MASK
+      type = RECORD_TYPES[type] || type
+      single_record = (type_and_flag & SINGLE_RECORD_MASK) > 0
+      case type
+      when :MULTI_REC_END, :DUMMY_RECORD
+        { :type => type }
+      else
         {
           :type           => type,
           :single_record  => single_record,
-          :content        => record_content(type, c.position),
-          :space          => c.get_ic_uint32,
-          :page_number    => c.get_ic_uint32,
+          :space          => c.name("space") { c.get_ic_uint32 },
+          :page_number    => c.name("page_number") { c.get_ic_uint32 },
         }
       end
     end
@@ -179,6 +145,6 @@ class Innodb::LogBlock
 
     puts
     puts "record:"
-    pp record
+    pp first_record_preamble
   end
 end

data/lib/innodb/page.rb

@@ -61,18 +61,12 @@ class Innodb::Page
     @size ||= @buffer.size
   end
 
-  # A helper function to return bytes from the page buffer based on offset
-  # and length, both in bytes.
-  def data(offset, length)
-    @buffer[offset...(offset + length)]
-  end
-
   # If no block is passed, return an Innodb::Cursor object positioned at a
   # specific offset. If a block is passed, create a cursor at the provided
   # offset and yield it to the provided block one time, and then return the
   # return value of the block.
   def cursor(offset)
-    new_cursor = Innodb::Cursor.new(self, offset)
+    new_cursor = Innodb::Cursor.new(@buffer, offset)
 
     if block_given?
       # Call the block once and return its return value.

data/lib/innodb/page/index.rb

@@ -11,8 +11,6 @@ require "innodb/fseg_entry"
 # (the actual data) which grow ascending by offset, free space, the page
 # directory which grows descending by offset, and the FIL trailer.
 class Innodb::Page::Index < Innodb::Page
-  attr_accessor :record_describer
-
   # The size (in bytes) of the "next" pointer in each record header.
   RECORD_NEXT_SIZE = 2
 
@@ -228,6 +226,11 @@ class Innodb::Page::Index < Innodb::Page
     end
   end
 
+  # A helper function to return the index id.
+  def index_id
+    page_header && page_header[:index_id]
+  end
+
   # A helper function to return the page level from the "page" header, for
   # easier access.
   def level
@@ -429,7 +432,7 @@ class Innodb::Page::Index < Innodb::Page
   def system_record(offset)
     cursor(offset).name("record[#{offset}]") do |c|
       header = c.peek { record_header(c) }
-      Innodb::Record.new({
+      Innodb::Record.new(self, {
         :offset => offset,
         :header => header,
         :next => header[:next],
@@ -448,6 +451,23 @@ class Innodb::Page::Index < Innodb::Page
     @supremum ||= system_record(pos_supremum)
   end
 
+  def record_describer=(o)
+    @record_describer = o
+  end
+
+  def record_describer
+    return @record_describer if @record_describer
+
+    if space and space.innodb_system and index_id
+      @record_describer =
+        space.innodb_system.data_dictionary.record_describer_by_index_id(index_id)
+    elsif space
+      @record_describer = space.record_describer
+    end
+
+    @record_describer
+  end
+
   # Return a set of field objects that describe the record.
   def make_record_description
     description = record_describer.description
@@ -559,48 +579,304 @@ class Innodb::Page::Index < Innodb::Page
         end
       end
 
-      Innodb::Record.new(this_record)
+      Innodb::Record.new(self, this_record)
+    end
+  end
+
+  # Return an array of row offsets for all entries in the page directory.
+  def directory
+    return @directory if @directory
+
+    @directory = []
+    cursor(pos_directory).backward.name("page_directory") do |c|
+      directory_slots.times do |n|
+        @directory.push c.name("slot[#{n}]") { c.get_uint16 }
+      end
+    end
+
+    @directory
+  end
+
+  # Return the slot number of the provided offset in the page directory, or nil
+  # if the offset is not present in the page directory.
+  def offset_is_directory_slot?(offset)
+    directory.index(offset)
+  end
+
+  # Return the slot number of the provided record in the page directory, or nil
+  # if the record is not present in the page directory.
+  def record_is_directory_slot?(this_record)
+    offset_is_directory_slot?(this_record.offset)
+  end
+
+  # Return the slot number for the page directory entry which "owns" the
+  # provided record. This will be either the record itself, or the nearest
+  # record with an entry in the directory and a value greater than the record.
+  def directory_slot_for_record(this_record)
+    if slot = record_is_directory_slot?(this_record)
+      return slot
     end
+
+    unless search_cursor = record_cursor(this_record.next)
+      raise "Couldn't position cursor"
+    end
+
+    while rec = search_cursor.record
+      if slot = record_is_directory_slot?(rec)
+        return slot
+      end
+    end
+
+    return record_is_directory_slot?(supremum)
   end
 
   # A class for cursoring through records starting from an arbitrary point.
   class RecordCursor
-    def initialize(page, offset)
-      @page   = page
-      @offset = offset
+    def initialize(page, offset, direction)
+      Innodb::Stats.increment :page_record_cursor_create
+
+      @initial = true
+      @page = page
+      @direction = direction
+      case offset
+      when :min
+        @record = @page.min_record
+      when :max
+        @record = @page.max_record
+      else
+        # Offset is a byte offset of a record (hopefully).
+        @record = @page.record(offset)
+      end
+    end
+
+    # Return the current record, mostly as a helper.
+    def current_record
+      @record
     end
 
     # Return the next record, and advance the cursor. Return nil when the
-    # end of records is reached.
-    def record
-      return nil unless @offset
+    # end of records (supremum) is reached.
+    def next_record
+      Innodb::Stats.increment :page_record_cursor_next_record
 
-      record = @page.record(@offset)
+      @record = @page.record(@record.next)
 
-      if record == @page.supremum
+      if @record == @page.supremum
         # We've reached the end of the linked list at supremum.
-        @offset = nil
-      elsif record.next == @offset
-        # The record links to itself; go ahead and return it (once), but set
-        # the next offset to nil to end the loop.
-        @offset = nil
-        record
+        nil
       else
-        @offset = record.next
-        record
+        @record
+      end
+    end
+
+    # Return the previous record, and advance the cursor. Return nil when the
+    # end of records (infimum) is reached.
+    def prev_record
+      Innodb::Stats.increment :page_record_cursor_prev_record
+
+      unless slot = @page.directory_slot_for_record(@record)
+        raise "Couldn't find slot for record"
+      end
+
+      unless search_cursor = @page.record_cursor(@page.directory[slot-1])
+        raise "Couldn't position search cursor"
+      end
+
+      while rec = search_cursor.record and rec.offset != @record.offset
+        if rec.next == @record.offset
+          if rec == @page.infimum
+            return nil
+          end
+          return @record = rec
+        end
+      end
+    end
+
+    # Return the next record in the order defined when the cursor was created.
+    def record
+      if @initial
+        @initial = false
+        return current_record
+      end
+
+      case @direction
+      when :forward
+        next_record
+      when :backward
+        prev_record
+      end
+    end
+
+    # Iterate through all records in the cursor.
+    def each_record
+      unless block_given?
+        return enum_for(:each_record)
+      end
+
+      while rec = record
+        yield rec
       end
     end
   end
 
   # Return a RecordCursor starting at offset.
-  def record_cursor(offset)
-    RecordCursor.new(self, offset)
+  def record_cursor(offset=:min, direction=:forward)
+    RecordCursor.new(self, offset, direction)
   end
 
-  # Return the first record on this page.
-  def first_record
-    first = record(infimum.next)
-    first if first != supremum
+  # Return the minimum record on this page.
+  def min_record
+    min = record(infimum.next)
+    min if min != supremum
+  end
+
+  # Return the maximum record on this page.
+  def max_record
+    # Since the records are only singly-linked in the forward direction, in
+    # order to do find the last record, we must create a cursor and walk
+    # backwards one step.
+    unless max_cursor = record_cursor(supremum.offset, :backward)
+      raise "Couldn't position cursor"
+    end
+    # Note the deliberate use of prev_record rather than record; we want
+    # to skip over supremum itself.
+    max = max_cursor.prev_record
+    max if max != infimum
+  end
+
+  # Search for a record within a single page, and return either a perfect
+  # match for the key, or the last record closest to they key but not greater
+  # than the key. (If an exact match is desired, compare_key must be used to
+  # check if the returned record matches. This makes the function useful for
+  # search in both leaf and non-leaf pages.)
+  def linear_search_from_cursor(search_cursor, key)
+    Innodb::Stats.increment :linear_search_from_cursor
+
+    this_rec = search_cursor.record
+
+    if Innodb.debug?
+      puts "linear_search_from_cursor: page=%i, level=%i, start=(%s)" % [
+        offset,
+        level,
+        this_rec && this_rec.key_string,
+      ]
+    end
+
+    # Iterate through all records until finding either a matching record or
+    # one whose key is greater than the desired key.
+    while this_rec && next_rec = search_cursor.record
+      Innodb::Stats.increment :linear_search_from_cursor_record_scans
+
+      if Innodb.debug?
+        puts "linear_search_from_cursor: page=%i, level=%i, current=(%s)" % [
+          offset,
+          level,
+          this_rec && this_rec.key_string,
+        ]
+      end
+
+      # If we reach supremum, return the last non-system record we got.
+      return this_rec if next_rec.header[:type] == :supremum
+
+      if this_rec.compare_key(key) < 0
+        return this_rec
+      end
+
+      if (this_rec.compare_key(key) >= 0) &&
+        (next_rec.compare_key(key) < 0)
+        # The desired key is either an exact match for this_rec or is greater
+        # than it but less than next_rec. If this is a non-leaf page, that
+        # will mean that the record will fall on the leaf page this node
+        # pointer record points to, if it exists at all.
+        return this_rec
+      end
+
+      this_rec = next_rec
+    end
+
+    this_rec
+  end
+
+  # Search or a record within a single page using the page directory to limit
+  # the number of record comparisons required. Once the last page directory
+  # entry closest to but not greater than the key is found, fall back to
+  # linear search using linear_search_from_cursor to find the closest record
+  # whose key is not greater than the desired key. (If an exact match is
+  # desired, the returned record must be checked in the same way as the above
+  # linear_search_from_cursor function.)
+  def binary_search_by_directory(dir, key)
+    Innodb::Stats.increment :binary_search_by_directory
+
+    return nil if dir.empty?
+
+    # Split the directory at the mid-point (using integer math, so the division
+    # is rounding down). Retrieve the record that sits at the mid-point.
+    mid = ((dir.size-1) / 2)
+    rec = record(dir[mid])
+
+    if Innodb.debug?
+      puts "binary_search_by_directory: page=%i, level=%i, dir.size=%i, dir[%i]=(%s)" % [
+        offset,
+        level,
+        dir.size,
+        mid,
+        rec.key_string,
+      ]
+    end
+
+    # The mid-point record was the infimum record, which is not comparable with
+    # compare_key, so we need to just linear scan from here. If the mid-point
+    # is the beginning of the page there can't be many records left to check
+    # anyway.
+    if rec.header[:type] == :infimum
+      return linear_search_from_cursor(record_cursor(rec.next), key)
+    end
+
+    # Compare the desired key to the mid-point record's key.
+    case rec.compare_key(key)
+    when 0
+      # An exact match for the key was found. Return the record.
+      Innodb::Stats.increment :binary_search_by_directory_exact_match
+      rec
+    when +1
+      # The mid-point record's key is less than the desired key.
+      if dir.size > 2
+        # There are more entries remaining from the directory, recurse again
+        # using binary search on the right half of the directory, which
+        # represents values greater than or equal to the mid-point record's
+        # key.
+        Innodb::Stats.increment :binary_search_by_directory_recurse_right
+        binary_search_by_directory(dir[mid...dir.size], key)
+      else
+        next_rec = record(dir[mid+1])
+        next_key = next_rec && next_rec.compare_key(key)
+        if dir.size == 1 || next_key == -1 || next_key == 0
+          # This is the last entry remaining from the directory, or our key is
+          # greater than rec and less than rec+1's key. Use linear search to
+          # find the record starting at rec.
+          Innodb::Stats.increment :binary_search_by_directory_linear_search
+          linear_search_from_cursor(record_cursor(rec.offset), key)
+        elsif next_key == +1
+          Innodb::Stats.increment :binary_search_by_directory_linear_search
+          linear_search_from_cursor(record_cursor(next_rec.offset), key)
+        else
+          nil
+        end
+      end
+    when -1
+      # The mid-point record's key is greater than the desired key.
+      if dir.size == 1
+        # If this is the last entry remaining from the directory, we didn't
+        # find anything workable.
+        Innodb::Stats.increment :binary_search_by_directory_empty_result
+        nil
+      else
+        # Recurse on the left half of the directory, which represents values
+        # less than the mid-point record's key.
+        Innodb::Stats.increment :binary_search_by_directory_recurse_left
+        binary_search_by_directory(dir[0...mid], key)
+      end
+    end
   end
 
   # Iterate through all records.
@@ -618,6 +894,7 @@ class Innodb::Page::Index < Innodb::Page
     nil
   end
 
+  # Iterate through all records in the garbage list.
   def each_garbage_record
     unless block_given?
       return enum_for(:each_garbage_record)
@@ -653,19 +930,6 @@ class Innodb::Page::Index < Innodb::Page
     nil
   end
 
-  # Return an array of row offsets for all entries in the page directory.
-  def directory
-    return @directory if @directory
-
-    @directory = []
-    cursor(pos_directory).backward.name("page_directory") do |c|
-      directory_slots.times do |n|
-        @directory.push c.name("slot[#{n}]") { c.get_uint16 }
-      end
-    end
-
-    @directory
-  end
 
   # Dump the contents of a page for debugging purposes.
   def dump