innodb_ruby 0.9.0 → 0.9.5

data/lib/innodb/page.rb

@@ -1,7 +1,5 @@
 # -*- encoding : utf-8 -*-
 
-require "innodb/cursor"
-
 # A generic class for any type of page, which handles reading the common
 # FIL header and trailer, and can handle (via #parse) dispatching to a more
 # specialized class depending on page type (which comes from the FIL header).
@@ -61,12 +59,26 @@ class Innodb::Page
     @size ||= @buffer.size
   end
 
-  # If no block is passed, return an Innodb::Cursor object positioned at a
+  # Return a simple string to uniquely identify this page within the space.
+  # Be careful not to call anything which would instantiate a BufferCursor
+  # so that we can use this method in cursor initialization.
+  def name
+    page_offset = BinData::Uint32be.read(@buffer.slice(4, 4))
+    page_type = BinData::Uint16be.read(@buffer.slice(24, 2))
+    "%i,%s" % [
+      page_offset,
+      PAGE_TYPE_BY_VALUE[page_type],
+    ]
+  end
+
+  # If no block is passed, return an BufferCursor object positioned at a
   # specific offset. If a block is passed, create a cursor at the provided
   # offset and yield it to the provided block one time, and then return the
   # return value of the block.
-  def cursor(offset)
-    new_cursor = Innodb::Cursor.new(@buffer, offset)
+  def cursor(buffer_offset)
+    new_cursor = BufferCursor.new(@buffer, buffer_offset)
+    new_cursor.push_name("space[#{space.name}]")
+    new_cursor.push_name("page[#{name}]")
 
     if block_given?
       # Call the block once and return its return value.
@@ -277,6 +289,28 @@ class Innodb::Page
     checksum != calculate_checksum
   end
 
+  def each_region
+    unless block_given?
+      return enum_for(:each_region)
+    end
+
+    yield({
+      :offset => pos_fil_header,
+      :length => size_fil_header,
+      :name   => :fil_header,
+      :info   => "FIL Header",
+    })
+
+    yield({
+      :offset => pos_fil_trailer,
+      :length => size_fil_trailer,
+      :name   => :fil_trailer,
+      :info   => "FIL Trailer",
+    })
+
+    nil
+  end
+
   # Implement a custom inspect method to avoid irb printing the contents of
   # the page buffer, since it's very large and mostly not interesting.
   def inspect

data/lib/innodb/page/blob.rb

@@ -41,6 +41,32 @@ class Innodb::Page::Blob < Innodb::Page
     end
   end
 
+  def each_region
+    unless block_given?
+      return enum_for(:each_region)
+    end
+
+    super do |region|
+      yield region
+    end
+
+    yield({
+      :offset => pos_blob_header,
+      :length => size_blob_header,
+      :name => :blob_header,
+      :info => "Blob Header",
+    })
+
+    yield({
+      :offset => pos_blob_data,
+      :length => blob_header[:length],
+      :name => :blob_data,
+      :info => "Blob Data",
+    })
+
+    nil
+  end
+
   # Dump the contents of a page for debugging purposes.
   def dump
     super

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -13,16 +13,12 @@ require "innodb/xdes"
 # The basic structure of FSP_HDR and XDES pages is: FIL header, FSP header,
 # an array of 256 XDES entries, empty (unused) space, and FIL trailer.
 class Innodb::Page::FspHdrXdes < Innodb::Page
+  extend ReadBitsAtOffset
+
   # A value added to the adjusted exponent stored in the page size field of
   # the flags in the FSP header.
   FLAGS_PAGE_SIZE_ADJUST = 9
 
-  # Read a given number of bits from an integer at a specific bit offset. The
-  # value returned is 0-based so does not need further shifting or adjustment.
-  def self.read_bits_at_offset(data, bits, offset)
-    ((data & (((1 << bits) - 1) << offset)) >> offset)
-  end
-
   # Decode the "flags" field in the FSP header, returning a hash of useful
   # decoded flags. Unfortunately, InnoDB has a fairly weird and broken
   # implementation of these flags. The flags are:
@@ -74,6 +70,14 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
     size / space.pages_per_extent
   end
 
+  def size_xdes_entry
+    @size_xdes_entry ||= Innodb::Xdes.new(self, cursor(pos_xdes_array)).size_entry
+  end
+
+  def size_xdes_array
+    entries_in_xdes_array * size_xdes_entry
+  end
+
   # Read the FSP (filespace) header, which contains a few counters and flags,
   # as well as list base nodes for each list maintained in the filespace.
   def fsp_header
@@ -134,6 +138,34 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
     end
   end
 
+  def each_region
+    unless block_given?
+      return enum_for(:each_region)
+    end
+
+    super do |region|
+      yield region
+    end
+
+    yield({
+      :offset => pos_fsp_header,
+      :length => size_fsp_header,
+      :name => :fsp_header,
+      :info => "FSP Header",
+    })
+
+    each_xdes do |xdes|
+      yield({
+        :offset => xdes.offset,
+        :length => size_xdes_entry,
+        :name => :xdes,
+        :info => "Extent Descriptor",
+      })
+    end
+
+    nil
+  end
+
   # Dump the contents of a page for debugging purposes.
   def dump
     super

data/lib/innodb/page/index.rb

@@ -31,6 +31,11 @@ class Innodb::Page::Index < Innodb::Page
   # "compact" record format.
   RECORD_COMPACT_BITS_SIZE = 3
 
+  # Maximum number of fields.
+  RECORD_MAX_N_SYSTEM_FIELDS  = 3
+  RECORD_MAX_N_FIELDS         = 1024 - 1
+  RECORD_MAX_N_USER_FIELDS    = RECORD_MAX_N_FIELDS - RECORD_MAX_N_SYSTEM_FIELDS * 2
+
   # Page direction values possible in the page_header's :direction field.
   PAGE_DIRECTION = {
     1 => :left,           # Inserts have been in descending order.
@@ -312,6 +317,8 @@ class Innodb::Page::Index < Innodb::Page
       when :redundant
         record_header_redundant_additional(header, cursor)
       end
+
+      header[:length] = origin - cursor.position
     end
 
     header
@@ -325,13 +332,13 @@ class Innodb::Page::Index < Innodb::Page
       # bit vector indicating NULL fields and the length of each
       # non-NULL variable-length field.
       if record_format
-        header[:field_nulls] = cursor.name("field_nulls") {
+        header[:nulls] = cursor.name("nulls") {
           record_header_compact_null_bitmap(cursor)
         }
-        header[:field_lengths], header[:field_externs] =
-          cursor.name("field_lengths_and_externs") {
+        header[:lengths], header[:externs] =
+          cursor.name("lengths_and_externs") {
             record_header_compact_variable_lengths_and_externs(cursor,
-              header[:field_nulls])
+              header[:nulls])
           }
       end
     end
@@ -339,39 +346,35 @@ class Innodb::Page::Index < Innodb::Page
 
   # Return an array indicating which fields are null.
   def record_header_compact_null_bitmap(cursor)
-    fields = (record_format[:key] + record_format[:row])
+    fields = record_fields
 
     # The number of bits in the bitmap is the number of nullable fields.
     size = fields.count { |f| f.nullable? }
 
     # There is no bitmap if there are no nullable fields.
-    return nil unless size > 0
-
-    # To simplify later checks, expand bitmap to one for each field.
-    bitmap = Array.new(fields.last.position + 1, false)
+    return [] unless size > 0
 
     null_bit_array = cursor.get_bit_array(size).reverse!
 
-    # For every nullable field, set whether the field is actually null.
-    fields.each do |f|
-      bitmap[f.position] = f.nullable? ? (null_bit_array.shift == 1) : false
+    # For every nullable field, select the ones which are actually null.
+    fields.inject([]) do |nulls, f|
+      nulls << f.name if f.nullable? && (null_bit_array.shift == 1)
+      nulls
     end
-
-    return bitmap
   end
 
   # Return an array containing an array of the length of each variable-length
   # field and an array indicating which fields are stored externally.
-  def record_header_compact_variable_lengths_and_externs(cursor, null_bitmap)
+  def record_header_compact_variable_lengths_and_externs(cursor, nulls)
     fields = (record_format[:key] + record_format[:row])
 
-    len_array = Array.new(fields.last.position + 1, 0)
-    ext_array = Array.new(fields.last.position + 1, false)
+    lengths = {}
+    externs = []
 
     # For each non-NULL variable-length field, the record header contains
     # the length in one or two bytes.
     fields.each do |f|
-      next if !f.variable? or (null_bitmap && null_bitmap[f.position])
+      next if !f.variable? || nulls.include?(f.name)
 
       len = cursor.get_uint8
       ext = false
@@ -383,18 +386,16 @@ class Innodb::Page::Index < Innodb::Page
         len = ((len & 0x3f) << 8) + cursor.get_uint8
       end
 
-      len_array[f.position] = len
-      ext_array[f.position] = ext
+      lengths[f.name] = len
+      externs << f.name if ext
     end
 
-    return len_array, ext_array
+    return lengths, externs
   end
 
   # Read additional header information from a redundant format record header.
   def record_header_redundant_additional(header, cursor)
-    header[:field_lengths] = []
-    header[:field_nulls] = []
-    header[:field_externs] = []
+    lengths, nulls, externs = [], [], []
 
     field_offsets = record_header_redundant_field_end_offsets(header, cursor)
 
@@ -403,17 +404,31 @@ class Innodb::Page::Index < Innodb::Page
       case header[:offset_size]
       when 1
         next_field_offset = (n & RECORD_REDUNDANT_OFF1_OFFSET_MASK)
-        header[:field_lengths]  << (next_field_offset - this_field_offset)
-        header[:field_nulls]    << ((n & RECORD_REDUNDANT_OFF1_NULL_MASK) != 0)
-        header[:field_externs]  << false
+        lengths << (next_field_offset - this_field_offset)
+        nulls   << ((n & RECORD_REDUNDANT_OFF1_NULL_MASK) != 0)
+        externs << false
       when 2
         next_field_offset = (n & RECORD_REDUNDANT_OFF2_OFFSET_MASK)
-        header[:field_lengths]  << (next_field_offset - this_field_offset)
-        header[:field_nulls]    << ((n & RECORD_REDUNDANT_OFF2_NULL_MASK) != 0)
-        header[:field_externs]  << ((n & RECORD_REDUNDANT_OFF2_EXTERN_MASK) != 0)
+        lengths << (next_field_offset - this_field_offset)
+        nulls   << ((n & RECORD_REDUNDANT_OFF2_NULL_MASK) != 0)
+        externs << ((n & RECORD_REDUNDANT_OFF2_EXTERN_MASK) != 0)
       end
       this_field_offset = next_field_offset
     end
+
+    # If possible, refer to fields by name rather than position for
+    # better formatting (i.e. pp).
+    if record_format
+      header[:lengths], header[:nulls], header[:externs] = {}, [], []
+
+      record_fields.each do |f|
+        header[:lengths][f.name] = lengths[f.position]
+        header[:nulls] << f.name if nulls[f.position]
+        header[:externs] << f.name if externs[f.position]
+      end
+    else
+      header[:lengths], header[:nulls], header[:externs] = lengths, nulls, externs
+    end
   end
 
   # Read field end offsets from the provided cursor for each field as counted
@@ -437,6 +452,7 @@ class Innodb::Page::Index < Innodb::Page
         :header => header,
         :next => header[:next],
         :data => c.name("data") { c.get_bytes(size_mum_record) },
+        :length => c.position - offset,
       })
     end
   end
@@ -470,24 +486,28 @@ class Innodb::Page::Index < Innodb::Page
 
   # Return a set of field objects that describe the record.
   def make_record_description
+    position = (0..RECORD_MAX_N_FIELDS).each
     description = record_describer.description
-
-    position = 0
-    fields = {:type => description[:type], :key => [], :row => []}
+    fields = {:type => description[:type], :key => [], :sys => [], :row => []}
 
     description[:key].each do |field|
-      fields[:key] << Innodb::Field.new(position, field[:name], *field[:type])
-      position += 1
+      fields[:key] << Innodb::Field.new(position.next, field[:name], *field[:type])
     end
 
-    if description[:type] == :clustered
-      # Account for TRX_ID and ROLL_PTR.
-      position += 2
+    # If this is a leaf page of the clustered index, read InnoDB's internal
+    # fields, a transaction ID and roll pointer.
+    if level == 0 && fields[:type] == :clustered
+      [["DB_TRX_ID", :TRX_ID,],["DB_ROLL_PTR", :ROLL_PTR]].each do |name, type|
+        fields[:sys] << Innodb::Field.new(position.next, name, type, :NOT_NULL)
+      end
     end
 
-    description[:row].each do |field|
-      fields[:row] << Innodb::Field.new(position, field[:name], *field[:type])
-      position += 1
+    # If this is a leaf page of the clustered index, or any page of a
+    # secondary index, read the non-key fields.
+    if (level == 0 && fields[:type] == :clustered) || (fields[:type] == :secondary)
+      description[:row].each do |field|
+        fields[:row] << Innodb::Field.new(position.next, field[:name], *field[:type])
+      end
     end
 
     fields
@@ -500,6 +520,13 @@ class Innodb::Page::Index < Innodb::Page
     end
   end
 
+  # Returns the (ordered) set of fields that describe records in this page.
+  def record_fields
+    if record_format
+      record_format.values_at(:key, :sys, :row).flatten.sort_by {|f| f.position}
+    end
+  end
+
   # Parse and return a record at a given offset.
   def record(offset)
     return nil unless offset
@@ -520,53 +547,23 @@ class Innodb::Page::Index < Innodb::Page
       if record_format
         this_record[:type] = record_format[:type]
 
-        # Read the key fields present in all types of pages.
-        this_record[:key] = []
-        record_format[:key].each do |f|
-          c.name("key[#{f.name}]") do
-            this_record[:key] << {
-              :name => f.name,
-              :type => f.data_type.name,
-              :value => f.value(this_record, c),
-              :extern => f.extern(this_record, c),
-            }
-          end
-        end
-
-        # If this is a leaf page of the clustered index, read InnoDB's internal
-        # fields, a transaction ID and roll pointer.
-        if level == 0 && record_format[:type] == :clustered
-          this_record[:transaction_id] = c.name("transaction_id") { c.get_hex(6) }
-          c.name("roll_pointer") do
-            rseg_id_insert_flag = c.name("rseg_id_insert_flag") { c.get_uint8 }
-            this_record[:roll_pointer]   = {
-              :is_insert  => (rseg_id_insert_flag & 0x80) == 0x80,
-              :rseg_id    => rseg_id_insert_flag & 0x7f,
-              :undo_log   => c.name("undo_log") {
-                {
-                  :page   => c.name("page")   { c.get_uint32 },
-                  :offset => c.name("offset") { c.get_uint16 },
-                }
-              }
-            }
-          end
+        # Used to indicate whether a field is part of key/row/sys.
+        fmap = [:key, :row, :sys].inject({}) do |h, k|
+          this_record[k] = []
+          record_format[k].each { |f| h[f.position] = k }
+          h
         end
 
-        # If this is a leaf page of the clustered index, or any page of a
-        # secondary index, read the non-key fields.
-        if (level == 0 && record_format[:type] == :clustered) ||
-          (record_format[:type] == :secondary)
-          # Read the non-key fields.
-          this_record[:row] = []
-          record_format[:row].each do |f|
-            c.name("row[#{f.name}]") do
-              this_record[:row] << {
-                :name => f.name,
-                :type => f.data_type.name,
-                :value => f.value(this_record, c),
-                :extern => f.extern(this_record, c),
-              }
-            end
+        # Read the fields present in this record.
+        record_fields.each do |f|
+          p = fmap[f.position]
+          c.name("#{p.to_s}[#{f.name}]") do
+            this_record[p] << {
+              :name => f.name,
+              :type => f.data_type.name,
+              :value => f.value(c, this_record),
+              :extern => f.extern(c, this_record),
+            }.reject { |k, v| v.nil? }
           end
         end
 
@@ -577,6 +574,18 @@ class Innodb::Page::Index < Innodb::Page
           this_record[:child_page_number] =
             c.name("child_page_number") { c.get_uint32 }
         end
+
+        this_record[:length] = c.position - offset
+
+        # Add system field accessors for convenience.
+        this_record[:sys].each do |f|
+          case f[:name]
+          when "DB_TRX_ID"
+            this_record[:transaction_id] = f[:value]
+          when "DB_ROLL_PTR"
+            this_record[:roll_pointer] = f[:value]
+          end
+        end
       end
 
       Innodb::Record.new(self, this_record)
@@ -649,23 +658,20 @@ class Innodb::Page::Index < Innodb::Page
       end
     end
 
-    # Return the current record, mostly as a helper.
-    def current_record
-      @record
-    end
-
     # Return the next record, and advance the cursor. Return nil when the
     # end of records (supremum) is reached.
     def next_record
       Innodb::Stats.increment :page_record_cursor_next_record
 
-      @record = @page.record(@record.next)
+      rec = @page.record(@record.next)
 
-      if @record == @page.supremum
+      # The garbage record list's end is self-linked, so we must check for
+      # both supremum and the current record's offset.
+      if rec == @page.supremum || rec.offset == @record.offset
         # We've reached the end of the linked list at supremum.
         nil
       else
-        @record
+        @record = rec
       end
     end
 
@@ -696,7 +702,7 @@ class Innodb::Page::Index < Innodb::Page
     def record
       if @initial
         @initial = false
-        return current_record
+        return @record
       end
 
       case @direction
@@ -885,7 +891,7 @@ class Innodb::Page::Index < Innodb::Page
       return enum_for(:each_record)
     end
 
-    c = record_cursor(infimum.next)
+    c = record_cursor(:min)
 
     while rec = c.record
       yield rec
@@ -930,6 +936,80 @@ class Innodb::Page::Index < Innodb::Page
     nil
   end
 
+  def each_region
+    unless block_given?
+      return enum_for(:each_region)
+    end
+
+    super do |region|
+      yield region
+    end
+
+    yield({
+      :offset => pos_index_header,
+      :length => size_index_header,
+      :name   => :index_header,
+      :info   => "Index Header",
+    })
+
+    yield({
+      :offset => pos_fseg_header,
+      :length => size_fseg_header,
+      :name   => :fseg_header,
+      :info   => "File Segment Header",
+    })
+
+    yield({
+      :offset => pos_infimum - 5,
+      :length => size_mum_record + 5,
+      :name   => :infimum,
+      :info   => "Infimum",
+    })
+
+    yield({
+      :offset => pos_supremum - 5,
+      :length => size_mum_record + 5,
+      :name   => :supremum,
+      :info   => "Supremum",
+    })
+
+
+    directory_slots.times do |n|
+      yield({
+        :offset => pos_directory - (n * 2),
+        :length => 2,
+        :name   => :directory,
+        :info   => "Page Directory",
+      })
+    end
+
+    each_garbage_record do |record|
+      yield({
+        :offset => record.offset - record.header[:length],
+        :length => record.length + record.header[:length],
+        :name   => :garbage,
+        :info   => "Garbage",
+      })
+    end
+
+    each_record do |record|
+      yield({
+        :offset => record.offset - record.header[:length],
+        :length => record.header[:length],
+        :name   => :record_header,
+        :info   => "Record Header",
+      })
+
+      yield({
+        :offset => record.offset,
+        :length => record.length,
+        :name   => :record_data,
+        :info   => "Record Data",
+      })
+    end
+
+    nil
+  end
 
   # Dump the contents of a page for debugging purposes.
   def dump