innodb_ruby 0.7.11 → 0.7.12

data/bin/innodb_space

@@ -36,7 +36,7 @@ def print_xdes_list(list)
   list.each do |entry|
     puts "%-12i%-64s" % [
       entry.xdes[:start_page],
-      entry.each_page_status.inject("") { |bitmap, page_status|
+      entry.each_page_status.inject("") { |bitmap, (page_number, page_status)|
         bitmap += page_status[:free] ? "." : "#"
         bitmap
       },
@@ -151,29 +151,13 @@ def space_indexes(space)
 
   space.each_index do |index|
     index.each_fseg do |fseg_name, fseg|
-      fragments =
-        fseg[:frag_array].inject(0) { |c, i| c += 1 if i; c }
-
-      used =
-        fragments +
-        fseg[:not_full_n_used] +
-        space.pages_per_extent * fseg[:full].base[:length]
-
-      allocated =
-        fragments +
-        space.pages_per_extent * fseg[:full].base[:length] +
-        space.pages_per_extent * fseg[:not_full].base[:length] +
-        space.pages_per_extent * fseg[:free].base[:length]
-
-      fill_factor = allocated > 0 ? 100.0 * (used.to_f / allocated.to_f) : 0.0
-
       puts "%-12i%-12i%-12s%-12i%-12i%-12s" % [
         index.id,
         index.root.offset,
         fseg_name,
-        used,
-        allocated,
-        "%.2f%%" % fill_factor,
+        fseg.used_pages,
+        fseg.total_pages,
+        "%.2f%%" % fseg.fill_factor,
       ]
     end
   end
@@ -229,6 +213,134 @@ def space_extents(space)
   print_xdes_list(space.each_xdes)
 end
 
+def print_inode_summary(inode)
+  puts "INODE fseg_id=%d, pages=%d, frag=%d, full=%d, not_full=%d, free=%d" % [
+    inode.fseg_id,
+    inode.total_pages,
+    inode.frag_array_n_used,
+    inode.full.length,
+    inode.not_full.length,
+    inode.free.length,
+  ]
+end
+
+def print_inode_detail(inode)
+  puts "INODE fseg_id=%d, pages=%d, frag=%d pages (%s), full=%d extents (%s), not_full=%d extents (%s) (%d/%d pages used), free=%d extents (%s)" % [
+    inode.fseg_id,
+    inode.total_pages,
+    inode.frag_array_n_used,
+    inode.frag_array_pages.join(", "),
+    inode.full.length,
+    inode.full.each.to_a.map { |x| "#{x.start_page}-#{x.end_page}" }.join(", "),
+    inode.not_full.length,
+    inode.not_full.each.to_a.map { |x| "#{x.start_page}-#{x.end_page}" }.join(", "),
+    inode.not_full_n_used,
+    inode.not_full.length * inode.space.pages_per_extent,
+    inode.free.length,
+    inode.free.each.to_a.map { |x| "#{x.start_page}-#{x.end_page}" }.join(", "),
+  ]
+end
+
+def space_inodes_summary(space)
+  space.each_inode do |inode|
+    print_inode_summary(inode)
+  end
+end
+
+def space_inodes_detail(space)
+  space.each_inode do |inode|
+    print_inode_detail(inode)
+  end
+end
+
+def page_account(space, page_number)
+  puts "Accounting for page #{page_number}:"
+
+  if page_number > space.pages
+    puts "  Page does not exist."
+    return
+  end
+
+  page = space.page(page_number)
+  puts "  Page type is #{page.type}."
+
+  xdes = space.xdes_for_page(page_number)
+  puts "  Extent descriptor for pages %d-%d is at page %d, offset %d." % [
+    xdes.start_page,
+    xdes.end_page,
+    xdes.this[:page],
+    xdes.this[:offset],
+  ]
+
+  if xdes.allocated_to_fseg?
+    puts "  Extent is fully allocated to fseg #{xdes.fseg_id}."
+  else
+    puts "  Extent is not fully allocated to an fseg; may be a fragment extent."
+  end
+
+  xdes_status = xdes.page_status(page_number)
+  puts "  Page is marked as %s in extent descriptor." % [
+    xdes_status[:free] ? 'free' : 'used'
+  ]
+
+  space.each_xdes_list do |name, list|
+    if list.include? xdes
+      puts "  Extent is in #{name} list of space."
+    end
+  end
+
+  page_inode = nil
+  space.each_inode do |inode|
+    inode.each_list do |name, list|
+      if list.include? xdes
+        page_inode = inode
+        puts "  Extent is in #{name} list of fseg #{inode.fseg_id}."
+      end
+    end
+
+    if inode.frag_array.include? page_number
+      page_inode = inode
+      puts "  Page is in fragment array of fseg %d." % [
+        inode.fseg_id,
+      ]
+    end
+  end
+
+  space.each_index do |index|
+    index.each_fseg do |fseg_name, fseg|
+      if page_inode == fseg
+        puts "  Fseg is in #{fseg_name} fseg of index #{index.id}."
+        puts "  Index root is page #{index.root.offset}."
+      end
+    end
+  end
+
+  if space.system_space?
+    if page_inode == space.trx_sys.fseg
+      puts "  Fseg is trx_sys."
+    end
+
+    if page_inode == space.trx_sys.doublewrite[:fseg]
+      puts "  Fseg is doublewrite buffer."
+    end
+
+    space.data_dictionary.each_index do |table_name, index_name, index|
+      index.each_fseg do |fseg_name, fseg|
+        if page_inode == fseg
+          puts "  Index is #{table_name}.#{index_name} of data dictionary."
+        end
+      end
+    end
+
+    space.trx_sys.rsegs.each_with_index do |rseg_slot, index|
+      if page.fil_header[:space_id] == rseg_slot[:space_id] &&
+        page.fil_header[:offset] == rseg_slot[:page_number]
+          puts "  Page is a rollback segment in slot #{index}."
+      end
+    end
+  end
+end
+
 def page_directory_summary(page)
   puts "%-8s%-8s%-14s%-8s%s" % [
     "slot",
@@ -264,16 +376,15 @@ def index_fseg_lists(index, fseg_name)
 end
 
 def index_fseg_list_iterate(index, fseg_name, list_name)
-  unless index.fseg(fseg_name)
+  unless fseg = index.fseg(fseg_name)
     raise "File segment '#{fseg_name}' doesn't exist"
   end
 
-  fseg = index.fseg(fseg_name)
-  unless fseg[list_name] && fseg[list_name].is_a?(Innodb::List)
+  unless list = fseg.list(list_name)
     raise "List '#{list_name}' doesn't exist"
   end
 
-  print_xdes_list(fseg[list_name])
+  print_xdes_list(list)
 end
 
 def index_fseg_frag_pages(index, fseg_name)
@@ -444,9 +555,18 @@ The following modes are supported:
   space-extents
     Iterate through all extents, printing the extent descriptor bitmap.
 
+  space-inodes-summary
+    Iterate through all inodes, printing a short summary of each FSEG.
+
+  space-inodes-detail
+    Iterate through all inodes, printing a detailed report of each FSEG.
+
   page-dump
     Dump the contents of a page, using the Ruby pp ("pretty-print") module.
 
+  page-account
+    Account for a page's usage in FSEGs.
+
   page-directory-summary
     Summarize the record contents of the page directory in a page. If a record
     describer is available, the key of each record will be printed.
@@ -500,6 +620,7 @@ Signal.trap("PIPE") { exit }
 
 getopt_options = [
   [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
+  [ "--trace",            "-t",     GetoptLong::NO_ARGUMENT ],
   [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--page-size",        "-P",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--page",             "-p",     GetoptLong::REQUIRED_ARGUMENT ],
@@ -515,6 +636,8 @@ getopt.each do |opt, arg|
   case opt
   when "--help"
     usage 0
+  when "--trace"
+    Innodb::Cursor.trace!
   when "--mode"
     @options.mode = arg
   when "--file"
@@ -544,7 +667,9 @@ end
 space = Innodb::Space.new(@options.file, @options.page_size)
 
 if @options.describer
-  space.record_describer = Innodb::RecordDescriber.const_get(@options.describer)
+  unless space.record_describer = eval(@options.describer)
+    space.record_describer = Innodb::RecordDescriber.const_get(@options.describer)
+  end
 end
 
 if ARGV.empty?
@@ -561,6 +686,14 @@ ARGV.each do |mode|
     @options.pages.each do |page_number|
       space.page(page_number).dump
     end
+  when "page-account"
+    if @options.pages.empty?
+      usage 1, "Page numbers to dump must be provided with --page/-p"
+    end
+
+    @options.pages.each do |page_number|
+      page_account(space, page_number)
+    end
   when "page-directory-summary"
     if @options.pages.empty?
       usage 1, "Page numbers to dump must be provided with --page/-p"
@@ -595,6 +728,10 @@ ARGV.each do |mode|
     space_indexes(space)
   when "space-extents"
     space_extents(space)
+  when "space-inodes-summary"
+    space_inodes_summary(space)
+  when "space-inodes-detail"
+    space_inodes_detail(space)
   when "index-recurse"
     unless space.record_describer
       usage 1, "Record describer necessary for index recursion" 

data/lib/innodb.rb

@@ -1,18 +1,25 @@
+# -*- encoding : utf-8 -*-
 # A set of classes for parsing and working with InnoDB data files.
 module Innodb; end
 
 require "enumerator"
 
 require "innodb/version"
+require "innodb/checksum"
 require "innodb/page"
+require "innodb/page/blob"
 require "innodb/page/fsp_hdr_xdes"
 require "innodb/page/inode"
 require "innodb/page/index"
 require "innodb/page/trx_sys"
+require "innodb/page/sys"
+require "innodb/page/undo_log"
 require "innodb/record_describer"
 require "innodb/field"
 require "innodb/space"
+require "innodb/inode"
 require "innodb/index"
 require "innodb/log_block"
 require "innodb/log"
-require "innodb/xdes"
+require "innodb/undo_log"
+require "innodb/xdes"

data/lib/innodb/checksum.rb

@@ -0,0 +1,30 @@
+# -*- encoding : utf-8 -*-
+class Innodb::Checksum
+  MAX   = 0xFFFFFFFF.freeze
+  MASK1 = 1463735687.freeze
+  MASK2	= 1653893711.freeze
+
+  # This is derived from ut_fold_ulint_pair in include/ut0rnd.ic in the
+  # InnoDB source code. Since Ruby's Bignum class is *much* slower than its
+  # Fixnum class, we mask back to 32 bits to keep things from overflowing
+  # and being promoted to Bignum.
+  def self.fold_pair(n1, n2)
+    (((((((n1 ^ n2 ^ MASK2) << 8) & MAX) + n1) & MAX) ^ MASK1) + n2) & MAX
+  end
+
+  # Iterate through the provided enumerator, which is expected to return a
+  # Fixnum (or something coercible to it), and "fold" them together to produce
+  # a single value.
+  def self.fold_enumerator(enumerator)
+    fold = 0
+    enumerator.each do |byte|
+      fold = fold_pair(fold, byte)
+    end
+    fold
+  end
+
+  # A simple helper (and example) to fold a provided string.
+  def self.fold_string(string)
+    fold_enumerator(string.bytes)
+  end
+end

data/lib/innodb/cursor.rb

@@ -1,62 +1,156 @@
+# -*- encoding : utf-8 -*-
 require "bindata"
 
 # A cursor to walk through InnoDB data structures to read fields.
 class Innodb::Cursor
-  def initialize(buffer, offset)
+
+  # An entry in a stack of cursors. The cursor position, direction, and
+  # name array are each attributes of the current cursor stack and are
+  # manipulated together.
+  class StackEntry
+    attr_accessor :cursor
+    attr_accessor :position
+    attr_accessor :direction
+    attr_accessor :name
+
+    def initialize(cursor, position=0, direction=:forward, name=nil)
+      @cursor = cursor
+      @position = position
+      @direction = direction
+      @name = name || []
+    end
+
+    def dup
+      StackEntry.new(cursor, position, direction, name.dup)
+    end
+  end
+
+  @@tracing = false
+
+  # Enable tracing for all Innodb::Cursor objects.
+  def self.trace!(arg=true)
+    @@tracing = arg
+  end
+
+  # Initialize a cursor within a buffer at the given position.
+  def initialize(buffer, position)
     @buffer = buffer
-    @cursor = [ offset ]
-    @direction = :forward
+    @stack = [ StackEntry.new(self, position) ]
+
+    trace_with :print_trace
+  end
+
+  # Print a trace output for this cursor. The method is passed a cursor object,
+  # position, raw byte buffer, and array of names.
+  def print_trace(cursor, position, bytes, name)
+    slice_size = 16
+    bytes.each_slice(slice_size).each_with_index do |slice_bytes, slice_count|
+      puts "%06i %s %-32s  %s" % [
+        position + (slice_count * slice_size),
+        direction == :backward ? "←" : "→",
+        slice_bytes.map { |n| "%02x" % n }.join,
+        slice_count == 0 ? name.join(".") : "↵",
+      ]
+    end
+  end
+
+  # Set a Proc or method on self to trace with.
+  def trace_with(arg=nil)
+    if arg.nil?
+      @trace_proc = nil
+    elsif arg.class == Proc
+      @trace_proc = arg
+    elsif arg.class == Symbol
+      @trace_proc = lambda { |cursor, position, bytes, name| self.send(arg, cursor, position, bytes, name) }
+    else
+      raise "Don't know how to trace with #{arg}"
+    end
+  end
+
+  # Generate a trace record from the current cursor.
+  def trace(position, bytes, name)
+    @trace_proc.call(self, position, bytes, name) if @@tracing && @trace_proc
+  end
+
+  # The current cursor object; the top of the stack.
+  def current
+    @stack.last
+  end
+
+  # Set the field name.
+  def name(name_arg=nil)
+    if name_arg.nil?
+      return current.name
+    end
+
+    unless block_given?
+      raise "No block given"
+    end
+
+    current.name.push name_arg
+    ret = yield(self)
+    current.name.pop
+    ret
+  end
+
+  # Return the direction of the current cursor.
+  def direction(direction_arg=nil)
+    if direction_arg.nil?
+      return current.direction
+    end
+
+    current.direction = direction_arg
+    self
   end
 
   # Set the direction of the cursor to "forward".
   def forward
-    @direction = :forward
-    self
+    direction(:forward)
   end
 
   # Set the direction of the cursor to "backward".
   def backward
-    @direction = :backward
-    self
+    direction(:backward)
   end
 
   # Return the position of the current cursor.
   def position
-    @cursor[0]
+    current.position
   end
 
   # Move the current cursor to a new absolute position.
-  def seek(offset)
-    @cursor[0] = offset if offset
+  def seek(position)
+    current.position = position if position
     self
   end
 
   # Adjust the current cursor to a new relative position.
-  def adjust(relative_offset)
-    @cursor[0] += relative_offset
+  def adjust(relative_position)
+    current.position += relative_position
     self
   end
 
   # Save the current cursor position and start a new (nested, stacked) cursor.
-  def push(offset=nil)
-    @cursor.unshift(offset.nil? ? @cursor[0] : offset)
+  def push(position=nil)
+    @stack.push current.dup
+    seek(position)
     self
   end
 
   # Restore the last cursor position.
   def pop
-    raise "No cursors to pop" unless @cursor.size > 1
-    @cursor.shift
+    raise "No cursors to pop" unless @stack.size > 1
+    @stack.pop
     self
   end
 
   # Execute a block and restore the cursor to the previous position after
   # the block returns. Return the block's return value after restoring the
-  # cursor.
-  def peek
+  # cursor. Optionally seek to provided position before executing block.
+  def peek(position=nil)
     raise "No block given" unless block_given?
-    push
-    result = yield
+    push(position)
+    result = yield(self)
     pop
     result
   end
@@ -65,16 +159,17 @@ class Innodb::Cursor
   # position and adjust the cursor position by that amount.
   def read_and_advance(length)
     data = nil
-    #print "data(#{@cursor[0]}..."
-    case @direction
+    cursor_start = current.position
+    case current.direction
     when :forward
-      data = @buffer.data(@cursor[0], length)
+      data = @buffer.data(current.position, length)
       adjust(length)
     when :backward
       adjust(-length)
-      data = @buffer.data(@cursor[0], length)
+      data = @buffer.data(current.position, length)
     end
-    #puts "#{@cursor[0]}) = #{data.bytes.map { |n| "%02x" % n }.join}"
+
+    trace(cursor_start, data.bytes, current.name)
     data
   end
 
@@ -83,49 +178,68 @@ class Innodb::Cursor
     read_and_advance(length)
   end
 
+  def each_byte_as_uint8(length)
+    unless block_given?
+      return enum_for(:each_byte_as_uint8, length)
+    end
+
+    read_and_advance(length).bytes.each do |byte|
+      yield byte
+    end
+
+    nil
+  end
+
   # Return raw bytes as hex.
   def get_hex(length)
     read_and_advance(length).bytes.map { |c| "%02x" % c }.join
   end
 
   # Read an unsigned 8-bit integer.
-  def get_uint8(offset=nil)
-    seek(offset)
+  def get_uint8(position=nil)
+    seek(position)
     data = read_and_advance(1)
     BinData::Uint8.read(data)
   end
 
   # Read a big-endian unsigned 16-bit integer.
-  def get_uint16(offset=nil)
-    seek(offset)
+  def get_uint16(position=nil)
+    seek(position)
     data = read_and_advance(2)
     BinData::Uint16be.read(data)
   end
 
   # Read a big-endian signed 16-bit integer.
-  def get_sint16(offset=nil)
-    seek(offset)
+  def get_sint16(position=nil)
+    seek(position)
     data = read_and_advance(2)
     BinData::Int16be.read(data)
   end
 
   # Read a big-endian unsigned 24-bit integer.
-  def get_uint24(offset=nil)
-    seek(offset)
+  def get_uint24(position=nil)
+    seek(position)
     data = read_and_advance(3)
     BinData::Uint24be.read(data)
   end
 
   # Read a big-endian unsigned 32-bit integer.
-  def get_uint32(offset=nil)
-    seek(offset)
+  def get_uint32(position=nil)
+    seek(position)
     data = read_and_advance(4)
     BinData::Uint32be.read(data)
   end
 
+  # Read a big-endian unsigned 48-bit integer.
+  def get_uint48(position=nil)
+    seek(position)
+    data = read_and_advance(6)
+    BinData::Uint48be.read(data)
+  end
+
   # Read a big-endian unsigned 64-bit integer.
-  def get_uint64(offset=nil)
-    seek(offset)
+  def get_uint64(position=nil)
+    seek(position)
     data = read_and_advance(8)
     BinData::Uint64be.read(data)
   end
@@ -141,6 +255,8 @@ class Innodb::Cursor
       get_uint24
     when 4
       get_uint32
+    when 6
+      get_uint48
     when 8
       get_uint64
     else
@@ -148,6 +264,10 @@ class Innodb::Cursor
     end
   end
 
+  def get_uint_array_by_size(size, count)
+    (0...count).to_a.inject([]) { |a, n| a << get_uint_by_size(size); a }
+  end
+
   # Read an InnoDB-compressed unsigned 32-bit integer.
   def get_ic_uint32
     flag = peek { get_uint8 }
@@ -193,6 +313,12 @@ class Innodb::Cursor
     BinData::Int32be.read(data) ^ (-1 << 31)
   end
 
+  # Read an InnoDB-munged signed 48-bit integer.
+  def get_i_sint48
+    data = read_and_advance(6)
+    BinData::Int48be.read(data) ^ (-1 << 47)
+  end
+
   # Read an InnoDB-munged signed 64-bit integer.
   def get_i_sint64
     data = read_and_advance(8)
@@ -210,6 +336,8 @@ class Innodb::Cursor
       get_i_sint24
     when 4
       get_i_sint32
+    when 6
+      get_i_sint48
     when 8
       get_i_sint64
     else