innodb_ruby 0.9.8 → 0.9.9

data/AUTHORS.md

@@ -10,3 +10,4 @@ The primary authors of `innodb_ruby` have been:
 Special thanks to the following contributors who have taken an interest and contributed their fixes:
 
 * Andrew Gaul \<gaul@maginatics.com\>
+* Frédéric -lefred- Descamps \<lefred@percona.com\>

data/README.md

@@ -15,3 +15,5 @@ Various parts of this library and the tools included may have wildly differing m
 
 * Visit the [innodb_ruby mailing list on Google Groups](https://groups.google.com/d/forum/innodb_ruby) or email [innodb_ruby@googlegroups.com](mailto:innodb_ruby@googlegroups.com) — If you have questions about `innodb_ruby` or its usage.
 * See the [RubyGems page for innodb_ruby](http://rubygems.org/gems/innodb_ruby) — Gem packaged releases are published regularly to RubyGems.org, which also provides online documentation.
+
+[![Build Status](https://travis-ci.org/jeremycole/innodb_ruby.svg?branch=master)](https://travis-ci.org/jeremycole/innodb_ruby)

data/bin/innodb_space

@@ -178,6 +178,10 @@ def system_spaces(innodb_system)
     next unless space
     print_space_information.call(table_name, space)
   end
+
+  innodb_system.each_orphan do |table_name|
+    puts "%-43s (orphan/tmp)" % table_name
+  end
 end
 
 # Print the contents of the SYS_TABLES data dictionary table.
@@ -451,13 +455,15 @@ end
 # filled block colored based on the index the page is part of. Print a legend
 # for the colors used afterwards.
 def space_extents_illustrate(space)
+  width = space.pages_per_extent
   puts
-  puts "%12s ╭%-64s╮" % [ "Start Page", "─" * 64 ]
+  puts "%12s ╭%-#{width}s╮" % [ "Start Page", "─" * width ]
 
   identifiers = {}
+  count_by_identifier = Hash.new(0)
 
   space.each_xdes do |entry|
-    puts "%12i │%-64s│" % [
+    puts "%12i │%-#{width}s│" % [
       entry.xdes[:start_page],
       entry.each_page_status.inject("") { |bitmap, (page_number, page_status)|
         if page_number < space.pages
@@ -473,7 +479,9 @@ def space_extents_illustrate(space)
             if page.respond_to?(:index_id)
               identifier = page.index_id
               unless identifiers[identifier]
-                identifiers[identifier] = "Index #{page.index_id}"
+                identifiers[identifier] = (page.index_id == Innodb::IbufIndex::INDEX_ID) ?
+                  "Insert Buffer Index" :
+                  "Index #{page.index_id}"
                 if space.innodb_system
                   table, index = space.innodb_system.table_and_index_name_by_id(page.index_id)
                   if table && index
@@ -484,6 +492,11 @@ def space_extents_illustrate(space)
             end
           end
           bitmap += filled_block(used_fraction, identifier)
+          if used_fraction != 0.0
+            count_by_identifier[identifier] += 1
+          else
+            count_by_identifier[:free] += 1
+          end
         else
           bitmap += " "
         end
@@ -492,15 +505,33 @@ def space_extents_illustrate(space)
     ]
   end
 
-  puts "%12s ╰%-64s╯" % [ "", "─" * 64 ]
+  puts "%12s ╰%-#{width}s╯" % [ "", "─" * width ]
 
   puts
-  puts "Legend:"
-  puts "  %s %s" % [filled_block(1.0, nil), "System"]
+  puts "Legend (%s = 1 page):" % [filled_block(1.0, nil)]
+  puts "  %-62s %8s %8s" % [
+    "Page Type", "Pages", "Ratio"
+  ]
+  puts "  %s %-60s %8i %7.2f%%" % [
+    filled_block(1.0, nil),
+    "System",
+    count_by_identifier[nil],
+    100.0 * (count_by_identifier[nil].to_f / space.pages.to_f),
+  ]
   identifiers.sort.each do |identifier, description|
-    puts "  %s %s" % [filled_block(1.0, identifier), description]
+    puts "  %s %-60s %8i %7.2f%%" % [
+      filled_block(1.0, identifier),
+      description,
+      count_by_identifier[identifier],
+      100.0 * (count_by_identifier[identifier].to_f / space.pages.to_f),
+    ]
   end
-  puts "  %s %s" % [filled_block(0.0, nil), "Free space"]
+  puts "  %s %-60s %8i %7.2f%%" % [
+    filled_block(0.0, nil),
+    "Free space",
+    count_by_identifier[:free],
+    100.0 * (count_by_identifier[:free].to_f / space.pages.to_f),
+  ]
   puts
 end
 
@@ -572,7 +603,7 @@ def space_lsn_age_illustrate(space)
   end
 
   puts
-  puts "Legend:"
+  puts "Legend (%s = 1 page):" % [filled_block(1.0, nil)]
   puts "  %12s %s %-12s" % [
     "Min LSN",
     lsn_legend,
@@ -759,6 +790,7 @@ def page_illustrate(page)
   blocks = Array.new(page.size, " ")
   identifiers = {}
   identifier_sort = 0
+  count_by_identifier = Hash.new(0)
 
   page.each_region.sort { |a,b| a[:offset] <=> b[:offset]}.each do |region|
     region[:length].times do |n|
@@ -782,6 +814,7 @@ def page_illustrate(page)
         end
       end
       blocks[region[:offset] + n] = filled_block(fraction, identifier, BLOCK_CHARS_H)
+      count_by_identifier[identifier] += 1
     end
   end
 
@@ -795,14 +828,33 @@ def page_illustrate(page)
   puts "%12s ╰%-#{width}s╯" % [ "", "─" * width ]
 
   puts
-  puts "Legend:"
+  puts "Legend (%s = 1 byte):" % [filled_block(1.0, nil)]
+  puts "  %-32s %8s %8s" % [
+    "Region Type",
+    "Bytes",
+    "Ratio",
+  ]
   identifiers.sort { |a,b| a[1] <=> b[1] }.each do |identifier, description|
-    puts "  %s %s" % [
+    puts "  %s %-30s %8i %7.2f%%" % [
       filled_block(1.0, identifier),
-      description.gsub(/^<\d+>/, "")
+      description.gsub(/^<\d+>/, ""),
+      count_by_identifier[identifier],
+      100.0 * (count_by_identifier[identifier].to_f / page.size.to_f),
     ]
   end
-  puts "  %s %s" % [filled_block(0.0, nil), "Garbage"]
+  puts "  %s %-30s %8i %7.2f%%" % [
+    filled_block(0.0, nil),
+    "Garbage",
+    count_by_identifier[nil],
+    100.0 * (count_by_identifier[nil].to_f / page.size.to_f),
+  ]
+  free_space = page.size - count_by_identifier.inject(0) { |sum,(k,v)| sum + v }
+  puts "  %s %-30s %8i %7.2f%%" % [
+    " ",
+    "Free",
+    free_space,
+    100.0 * (free_space.to_f / page.size.to_f),
+  ]
 
   puts
 end
@@ -969,6 +1021,11 @@ The following options are supported:
   --help, -?
     Print this usage text.
 
+  --trace, -t
+    Enable tracing of all data read. Specify twice to enable even more
+    tracing (including reads during opening of the tablespace) which can
+    be quite noisy.
+
   --system-space-file, -s <file>
     Load the system tablespace file <file> (normally ibdata1).
 
@@ -997,10 +1054,6 @@ The following options are supported:
   --describer, -d <describer>
     Use the named record describer to parse records in index pages.
 
-  --page-size, -P <size>
-    Provide the page size, overriding auto-detection (in KiB): 16, 8, 4, 2, 1.
-    Page sizes other than 16 may not work well, or at all.
-
 The following modes are supported:
 
   system-spaces
@@ -1129,6 +1182,7 @@ Signal.trap("INT")  { exit }
 Signal.trap("PIPE") { exit }
 
 @options = OpenStruct.new
+@options.trace                  = 0
 @options.system_space_file      = nil
 @options.space_file             = nil
 @options.table_name             = nil
@@ -1136,7 +1190,6 @@ Signal.trap("PIPE") { exit }
 @options.page                   = nil
 @options.level                  = nil
 @options.list                   = nil
-@options.page_size              = nil
 @options.describer              = nil
 
 getopt_options = [
@@ -1150,7 +1203,6 @@ getopt_options = [
   [ "--level",                  "-l",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--list",                   "-L",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--require",                "-r",     GetoptLong::REQUIRED_ARGUMENT ],
-  [ "--page-size",              "-P",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--describer",              "-d",     GetoptLong::REQUIRED_ARGUMENT ],
 ]
 
@@ -1161,7 +1213,7 @@ getopt.each do |opt, arg|
   when "--help"
     usage 0
   when "--trace"
-    BufferCursor.trace!
+    @options.trace += 1
   when "--system-space-file"
     @options.system_space_file = arg
   when "--space-file"
@@ -1178,11 +1230,6 @@ getopt.each do |opt, arg|
     @options.list = arg.to_sym
   when "--require"
     require File.expand_path(arg)
-  when "--page-size"
-    unless [1, 2, 4, 8, 16].include?(arg.to_i)
-      usage 1, "Page size #{arg} is not understood"
-    end
-    @options.page_size = arg.to_i * 1024
   when "--describer"
     @options.describer = arg
   end
@@ -1196,6 +1243,10 @@ if @options.system_space_file and @options.space_file
   usage 1, "Only one of system space or space file may be specified"
 end
 
+if @options.trace > 1
+  BufferCursor.trace!
+end
+
 innodb_system = nil
 if @options.system_space_file
   innodb_system = Innodb::System.new(@options.system_space_file)
@@ -1205,7 +1256,7 @@ space = innodb_system ? innodb_system.system_space : nil
 if innodb_system and @options.table_name
   space = innodb_system.space_by_table_name(@options.table_name)
 elsif @options.space_file
-  space = Innodb::Space.new(@options.space_file, @options.page_size)
+  space = Innodb::Space.new(@options.space_file)
 end
 
 if @options.describer
@@ -1262,6 +1313,10 @@ if [
   usage 1, "Record describer must be specified using -d/--describer"
 end
 
+if @options.trace > 0
+  BufferCursor.trace!
+end
+
 case mode
 when "system-spaces"
   system_spaces(innodb_system)

data/lib/innodb/data_dictionary.rb

@@ -145,6 +145,7 @@ class Innodb::DataDictionary
     :UNIVERSAL => 4,
     :IBUF      => 8,
     :CORRUPT   => 16,
+    :FTS       => 32,
   }
 
   # A hash of INDEX_TYPE_FLAG keys by value.
@@ -228,6 +229,41 @@ class Innodb::DataDictionary
     system_space.data_dictionary_page.data_dictionary_header[:indexes]
   end
 
+  def data_dictionary_index_ids
+    if @data_dictionary_index_ids
+      return @data_dictionary_index_ids
+    end
+
+    @data_dictionary_index_ids = {}
+    data_dictionary_indexes.each do |table, indexes|
+      indexes.each do |index, root_page_number|
+        if root_page = system_space.page(root_page_number)
+          @data_dictionary_index_ids[root_page.index_id] = {
+            :table => table,
+            :index => index
+          }
+        end
+      end
+    end
+
+    @data_dictionary_index_ids
+  end
+
+  def data_dictionary_table?(table_name)
+    DATA_DICTIONARY_RECORD_DESCRIBERS.include?(table_name.to_sym)
+  end
+
+  def data_dictionary_index?(table_name, index_name)
+    return unless data_dictionary_table?(table_name)
+    DATA_DICTIONARY_RECORD_DESCRIBERS[table_name.to_sym].include?(index_name.to_sym)
+  end
+
+  def data_dictionary_index_describer(table_name, index_name)
+    return unless data_dictionary_index?(table_name, index_name)
+
+    DATA_DICTIONARY_RECORD_DESCRIBERS[table_name.to_sym][index_name.to_sym].new
+  end
+
   # Return an Innodb::Index object initialized to the
   # internal data dictionary index with an appropriate
   # record describer so that records can be recursed.
@@ -241,10 +277,9 @@ class Innodb::DataDictionary
     end
 
     # If we have a record describer for this index, load it.
-    record_describer = DATA_DICTIONARY_RECORD_DESCRIBERS[table_name] &&
-                       DATA_DICTIONARY_RECORD_DESCRIBERS[table_name][index_name]
+    record_describer = data_dictionary_index_describer(table_name, index_name)
 
-    system_space.index(index_root_page, record_describer.new)
+    system_space.index(index_root_page, record_describer)
   end
 
   # Iterate through all data dictionary indexes, yielding the
@@ -605,6 +640,10 @@ class Innodb::DataDictionary
   # Return an Innodb::RecordDescriber object describing records for a given
   # index by table name and index name.
   def record_describer_by_index_name(table_name, index_name)
+    if data_dictionary_index?(table_name, index_name)
+      return data_dictionary_index_describer(table_name, index_name)
+    end
+
     unless index = index_by_name(table_name, index_name)
       raise "Index #{index_name} for table #{table_name} not found"
     end
@@ -632,6 +671,10 @@ class Innodb::DataDictionary
   # Return an Innodb::RecordDescriber object describing the records
   # in a given index by index ID.
   def record_describer_by_index_id(index_id)
+    if dd_index = data_dictionary_index_ids[index_id]
+      return data_dictionary_index_describer(dd_index[:table], dd_index[:index])
+    end
+
     unless index = index_by_id(index_id)
       raise "Index #{index_id} not found"
     end

data/lib/innodb/fseg_entry.rb

@@ -22,9 +22,15 @@ class Innodb::FsegEntry
   # Return an INODE entry which represents this file segment.
   def self.get_inode(space, cursor)
     address = cursor.name("address") { get_entry_address(cursor) }
+    if address[:offset] == 0
+      return nil
+    end
+
     page = space.page(address[:page_number])
-    if page.type == :INODE
-      page.inode_at(page.cursor(address[:offset]))
+    if page.type != :INODE
+      return nil
     end
+
+    page.inode_at(page.cursor(address[:offset]))
   end
 end

data/lib/innodb/ibuf_bitmap.rb

@@ -0,0 +1,50 @@
+# -*- encoding : utf-8 -*-
+
+class Innodb::IbufBitmap
+  BITS_PER_PAGE = 4
+
+  BITMAP_BV_FREE     = 1 + 2
+  BITMAP_BV_BUFFERED = 4
+  BITMAP_BV_IBUF     = 8
+
+  BITMAP_BV_ALL =
+    BITMAP_BV_FREE |
+    BITMAP_BV_BUFFERED |
+    BITMAP_BV_IBUF
+
+  def initialize(page, cursor)
+    @page = page
+    @bitmap = read_bitmap(page, cursor)
+  end
+
+  def size_bitmap
+    (@page.space.pages_per_bookkeeping_page * BITS_PER_PAGE) / 8
+  end
+
+  def read_bitmap(page, cursor)
+    cursor.name("ibuf_bitmap") do |c|
+      c.get_bytes(size_bitmap)
+    end
+  end
+
+  def each_page_status
+    unless block_given?
+      return enum_for(:each_page_status)
+    end
+
+    bitmap = @bitmap.enum_for(:each_byte)
+
+    bitmap.each_with_index do |byte, byte_index|
+      (0..1).each do |page_offset|
+        page_number = (byte_index * 2) + page_offset
+        page_bits = ((byte >> (page_offset * BITS_PER_PAGE)) & BITMAP_BV_ALL)
+        page_status = {
+          :free => (page_bits & BITMAP_BV_FREE),
+          :buffered => (page_bits & BITMAP_BV_BUFFERED != 0),
+          :ibuf => (page_bits & BITMAP_BV_IBUF != 0),
+        }
+        yield page_number, page_status
+      end
+    end
+  end
+end

data/lib/innodb/ibuf_index.rb

@@ -0,0 +1,3 @@
+class Innodb::IbufIndex
+  INDEX_ID = 0xFFFFFFFF00000000
+end

data/lib/innodb/index.rb

@@ -3,6 +3,7 @@
 # An InnoDB index B-tree, given an Innodb::Space and a root page number.
 class Innodb::Index
   attr_reader :root
+  attr_reader :space
   attr_accessor :record_describer
 
   def initialize(space, root_page_number, record_describer=nil)

data/lib/innodb/page/blob.rb

@@ -2,7 +2,7 @@
 
 class Innodb::Page::Blob < Innodb::Page
   def pos_blob_header
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   def size_blob_header

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -17,40 +17,50 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
 
   # A value added to the adjusted exponent stored in the page size field of
   # the flags in the FSP header.
-  FLAGS_PAGE_SIZE_ADJUST = 9
+  FLAGS_PAGE_SIZE_SHIFT = 9
+
+  def self.shift_page_size(page_size_shifted)
+    if page_size_shifted != 0
+      (1 << (FLAGS_PAGE_SIZE_SHIFT + page_size_shifted))
+    end
+  end
 
   # Decode the "flags" field in the FSP header, returning a hash of useful
-  # decoded flags. Unfortunately, InnoDB has a fairly weird and broken
-  # implementation of these flags. The flags are:
+  # decodings of the flags (based on MySQl 5.6 definitions). The flags are:
   #
   # Offset    Size    Description
-  # 0         1       Page Format (redundant, compact). This is unfortunately
-  #                   coerced to 0 if it is "compact" and no other flags are
-  #                   set, making it useless to innodb_ruby.
+  # 0         1       Post-Antelope Flag.
   # 1         4       Compressed Page Size (zip_size). This is stored as a
   #                   power of 2, minus 9. Since 0 is reserved to mean "not
   #                   compressed", the minimum value is 1, thus making the
   #                   smallest page size 1024 (2 ** (9 + 1)).
-  # 5         1       Table Format (Antelope, Barracuda). This was supposed
-  #                   to reserve 6 bits, but due to a bug in InnoDB only
-  #                   actually reserved 1 bit.
+  # 5         1       Atomic Blobs Flag.
+  # 6         4       System Page Size (innodb_page_size, UNIV_PAGE_SIZE).
+  #                   The setting of the system page size when the tablespace
+  #                   was created, stored in the same format as the compressed
+  #                   page size above.
+  # 10        1       Data Directory Flag.
   #
   def self.decode_flags(flags)
-    # The page size for compressed pages is stored at bit offset 1 and consumes
-    # 4 bits. Value 0 means the page is not compressed.
-    page_size = read_bits_at_offset(flags, 4, 1)
+    system_page_size =
+      shift_page_size(read_bits_at_offset(flags, 4, 6)) ||
+      Innodb::Space::DEFAULT_PAGE_SIZE
+    compressed_page_size = shift_page_size(read_bits_at_offset(flags, 4, 1))
+
     {
-      :compressed => page_size == 0 ? false : true,
-      :page_size => page_size == 0 ?
-        Innodb::Space::DEFAULT_PAGE_SIZE :
-        (1 << (FLAGS_PAGE_SIZE_ADJUST + page_size)),
+      :system_page_size => system_page_size,
+      :compressed => compressed_page_size ? false : true,
+      :page_size => compressed_page_size || system_page_size,
+      :post_antelope => read_bits_at_offset(flags, 1, 0) == 1,
+      :atomic_blogs => read_bits_at_offset(flags, 1, 5) == 1,
+      :data_directory => read_bits_at_offset(flags, 1, 10) == 1,
       :value => flags,
     }
   end
 
   # The FSP header immediately follows the FIL header.
   def pos_fsp_header
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   # The FSP header contains six 32-bit integers, one 64-bit integer, and 5
@@ -87,7 +97,7 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
         :unused             => c.name("unused") { c.get_uint32 },
         :size               => c.name("size") { c.get_uint32 },
         :free_limit         => c.name("free_limit") { c.get_uint32 },
-        :flags              => c.name("flags") { 
+        :flags              => c.name("flags") {
           self.class.decode_flags(c.get_uint32)
         },
         :frag_n_used        => c.name("frag_n_used") { c.get_uint32 },
@@ -155,11 +165,12 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
     })
 
     each_xdes do |xdes|
+      state = xdes.state || "unused"
       yield({
         :offset => xdes.offset,
         :length => size_xdes_entry,
-        :name => :xdes,
-        :info => "Extent Descriptor",
+        :name => "xdes_#{state}".to_sym,
+        :info => "Extent Descriptor (#{state})",
       })
     end
 

data/lib/innodb/page/ibuf_bitmap.rb

@@ -0,0 +1,47 @@
+# -*- encoding : utf-8 -*-
+
+class Innodb::Page::IbufBitmap < Innodb::Page
+  extend ReadBitsAtOffset
+
+  def pos_ibuf_bitmap
+    pos_page_body
+  end
+
+  def size_ibuf_bitmap
+    (Innodb::IbufBitmap::BITS_PER_PAGE * space.pages_per_bookkeeping_page) / 8
+  end
+
+  def ibuf_bitmap
+    Innodb::IbufBitmap.new(self, cursor(pos_ibuf_bitmap))
+  end
+
+  def each_region
+    unless block_given?
+      return enum_for(:each_region)
+    end
+
+    super do |region|
+      yield region
+    end
+
+    yield({
+      :offset => pos_ibuf_bitmap,
+      :length => size_ibuf_bitmap,
+      :name => :ibuf_bitmap,
+      :info => "Insert Buffer Bitmap",
+    })
+
+    nil
+  end
+
+  def dump
+    super
+
+    puts "ibuf bitmap:"
+    ibuf_bitmap.each_page_status do |page_number, page_status|
+      puts "  Page %i: %s" % [page_number, page_status.inspect]
+    end
+  end
+end
+
+Innodb::Page::SPECIALIZED_CLASSES[:IBUF_BITMAP] = Innodb::Page::IbufBitmap

data/lib/innodb/page/index.rb

@@ -73,7 +73,7 @@ class Innodb::Page::Index < Innodb::Page
   # Return the byte offset of the start of the "index" page header, which
   # immediately follows the "fil" header.
   def pos_index_header
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   # The size of the "index" header.
@@ -287,7 +287,7 @@ class Innodb::Page::Index < Innodb::Page
         #   13 bits for heap_number
         bits1 = c.name("bits1") { c.get_uint16 }
         header[:type] = RECORD_TYPES[bits1 & 0x07]
-        header[:heap_number] = (bits1 & 0xf8) >> 3
+        header[:heap_number] = (bits1 & 0xfff8) >> 3
       when :redundant
         # The "next" pointer is an absolute offset within the page.
         header[:next] = c.name("next") { c.get_uint16 }

data/lib/innodb/page/inode.rb

@@ -9,7 +9,7 @@ class Innodb::Page::Inode < Innodb::Page
   # Return the byte offset of the list node, which immediately follows the
   # FIL header.
   def pos_list_entry
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   # Return the size of the list node.

data/lib/innodb/page/sys.rb

@@ -2,6 +2,7 @@
 
 require "innodb/page/sys_rseg_header"
 require "innodb/page/sys_data_dictionary_header"
+require "innodb/page/sys_ibuf_header"
 
 # Another layer of indirection for pages of type SYS, as they have multiple
 # uses within InnoDB. We'll override the self.handle method and check the
@@ -9,6 +10,8 @@ require "innodb/page/sys_data_dictionary_header"
 class Innodb::Page::Sys < Innodb::Page
   def self.handle(page, space, buffer)
     case
+    when page.offset == 3
+      Innodb::Page::SysIbufHeader.new(space, buffer)
     when page.offset == 7
       Innodb::Page::SysDataDictionaryHeader.new(space, buffer)
     when space.rseg_page?(page.offset)

data/lib/innodb/page/sys_data_dictionary_header.rb

@@ -3,7 +3,7 @@
 class Innodb::Page::SysDataDictionaryHeader < Innodb::Page
   # The position of the data dictionary header within the page.
   def pos_data_dictionary_header
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   # The size of the data dictionary header.

data/lib/innodb/page/sys_ibuf_header.rb

@@ -0,0 +1,45 @@
+# -*- encoding : utf-8 -*-
+
+class Innodb::Page::SysIbufHeader < Innodb::Page
+  def pos_ibuf_header
+    pos_page_body
+  end
+
+  def size_ibuf_header
+    Innodb::FsegEntry::SIZE
+  end
+
+  def ibuf_header
+    cursor(pos_ibuf_header).name("ibuf_header") do |c|
+      {
+        :fseg => c.name("fseg") {
+          Innodb::FsegEntry.get_inode(space, c)
+        }
+      }
+    end
+  end
+
+  def each_region
+    unless block_given?
+      return enum_for(:each_region)
+    end
+
+    super do |region|
+      yield region
+    end
+
+    yield({
+      :offset => pos_ibuf_header,
+      :length => size_ibuf_header,
+      :name => :ibuf_header,
+      :info => "Insert Buffer Header",
+    })
+  end
+
+  def dump
+    super
+
+    puts "ibuf header:"
+    pp ibuf_header
+  end
+end

data/lib/innodb/page/sys_rseg_header.rb

@@ -6,7 +6,7 @@ class Innodb::Page::SysRsegHeader < Innodb::Page
 
   # The position of the rollback segment header within the page.
   def pos_rseg_header
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   # The size of the rollback segment header.

data/lib/innodb/page/trx_sys.rb

@@ -12,7 +12,7 @@
 class Innodb::Page::TrxSys < Innodb::Page
   # The TRX_SYS header immediately follows the FIL header.
   def pos_trx_sys_header
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   def size_trx_sys_header

data/lib/innodb/page/undo_log.rb

@@ -2,7 +2,7 @@
 
 class Innodb::Page::UndoLog < Innodb::Page
   def pos_undo_page_header
-    pos_fil_header + size_fil_header
+    pos_page_body
   end
 
   def size_undo_page_header

data/lib/innodb/space.rb

@@ -22,18 +22,15 @@ class Innodb::Space
 
   # Open a space file, optionally providing the page size to use. Pages
   # that aren't 16 KiB may not be supported well.
-  def initialize(file, page_size=nil)
+  def initialize(file)
     @file = File.open(file)
     @size = @file.stat.size
 
-    if page_size
-      @page_size = page_size
-    else
-      @page_size = fsp_flags[:page_size]
-    end
+    @system_page_size = fsp_flags[:system_page_size]
+    @page_size        = fsp_flags[:page_size]
+    @compressed       = fsp_flags[:compressed]
 
     @pages = (@size / @page_size)
-    @compressed = fsp_flags[:compressed]
     @innodb_system = nil
     @record_describer = nil
   end
@@ -45,6 +42,9 @@ class Innodb::Space
   # this space.
   attr_accessor :record_describer
 
+  # The system default page size (in bytes), equivalent to UNIV_PAGE_SIZE.
+  attr_reader :system_page_size
+
   # The size (in bytes) of each page in the space.
   attr_reader :page_size
 
@@ -83,18 +83,24 @@ class Innodb::Space
   def raw_fsp_header_flags
     # A simple sanity check. The FIL header should be initialized in page 0,
     # to offset 0 and page type :FSP_HDR (8).
-    page_offset = BinData::Uint32be.read(read_at_offset(4, 4))
-    page_type   = BinData::Uint16be.read(read_at_offset(24, 2))
+    page_offset = BinData::Uint32be.read(read_at_offset(4, 4)).to_i
+    page_type   = BinData::Uint16be.read(read_at_offset(24, 2)).to_i
     unless page_offset == 0 && Innodb::Page::PAGE_TYPE_BY_VALUE[page_type] == :FSP_HDR
-      raise "Something is very wrong; Page 0 does not seem to be type FSP_HDR"
+      raise "Something is very wrong; Page 0 does not seem to be type FSP_HDR; got page type %i but expected %i" % [
+        page_type,
+        Innodb::Page::PAGE_TYPE[:FSP_HDR][:value],
+      ]
     end
 
     # Another sanity check. The Space ID should be the same in both the FIL
     # and FSP headers.
-    fil_space = BinData::Uint32be.read(read_at_offset(34, 4))
-    fsp_space = BinData::Uint32be.read(read_at_offset(38, 4))
+    fil_space = BinData::Uint32be.read(read_at_offset(34, 4)).to_i
+    fsp_space = BinData::Uint32be.read(read_at_offset(38, 4)).to_i
     unless fil_space == fsp_space
-      raise "Something is very wrong; FIL and FSP header Space IDs don't match"
+      raise "Something is very wrong; FIL and FSP header Space IDs don't match: FIL is %i but FSP is %i" % [
+        fil_space,
+        fsp_space,
+      ]
     end
 
     # Well, we're as sure as we can be. Read the flags field and decode it.
@@ -113,30 +119,56 @@ class Innodb::Space
     end
   end
 
-  # The size (in bytes) of an extent.
-  def extent_size
-    1048576
-  end
-
   # The number of pages per extent.
   def pages_per_extent
-    extent_size / page_size
+    # Note that uncompressed tables and compressed tables using the same page
+    # size will have a different number of pages per "extent" because InnoDB
+    # compression uses the FSP_EXTENT_SIZE define (which is then based on the
+    # UNIV_PAGE_SIZE define, which may be based on the innodb_page_size system
+    # variable) for compressed tables rather than something based on the actual
+    # compressed page size.
+    #
+    # For this reason, an "extent" differs in size as follows (the maximum page
+    # size supported for compressed tables is the innodb_page_size):
+    #
+    #   innodb_page_size                | innodb compression              |
+    #   page size | extent size | pages | page size | extent size | pages |
+    #   16384     | 1 MiB       | 64    | 16384     | 1 MiB       | 64    |
+    #                                   | 8192      | 512 KiB     | 64    |
+    #                                   | 4096      | 256 KiB     | 64    |
+    #                                   | 2048      | 128 KiB     | 64    |
+    #                                   | 1024      | 64 KiB      | 64    |
+    #   8192      | 1 MiB       | 128   | 8192      | 1 MiB       | 128   |
+    #                                   | 4096      | 512 KiB     | 128   |
+    #                                   | 2048      | 256 KiB     | 128   |
+    #                                   | 1024      | 128 KiB     | 128   |
+    #   4096      | 1 MiB       | 256   | 4096      | 1 MiB       | 256   |
+    #                                   | 2048      | 512 KiB     | 256   |
+    #                                   | 1024      | 256 KiB     | 256   |
+    #
+
+    1048576 / system_page_size
   end
 
-  # The number of pages per FSP_HDR/XDES page. This is crudely mapped to the
-  # page size, and works for pages down to 1KiB.
-  def pages_per_xdes_page
-    page_size
+  # The size (in bytes) of an extent.
+  def extent_size
+    pages_per_extent * page_size
   end
 
-  # An array of all FSP/XDES page numbers for the space.
-  def xdes_page_numbers
-    (0..(@pages / pages_per_xdes_page)).map { |n| n * pages_per_xdes_page }
+  # The number of pages per FSP_HDR/XDES/IBUF_BITMAP page. This is crudely
+  # mapped to the page size, and works for pages down to 1KiB.
+  def pages_per_bookkeeping_page
+    page_size
   end
 
   # The FSP_HDR/XDES page which will contain the XDES entry for a given page.
   def xdes_page_for_page(page_number)
-    page_number - (page_number % pages_per_xdes_page)
+    page_number - (page_number % pages_per_bookkeeping_page)
+  end
+
+  # The IBUF_BITMAP page which will contain the bitmap entry for a given page.
+  def ibuf_bitmap_page_for_page(page_number)
+    page_number - (page_number % pages_per_bookkeeping_page) + 1
   end
 
   # The XDES entry offset for a given page within its FSP_HDR/XDES page's
@@ -343,6 +375,17 @@ class Innodb::Space
     end
   end
 
+  # An array of all FSP/XDES page numbers for the space.
+  def each_xdes_page_number
+    unless block_given?
+      return enum_for(:each_xdes_page_number)
+    end
+
+    0.step(pages - 1, pages_per_bookkeeping_page).each do |n|
+      yield n
+    end
+  end
+
   # Iterate through all FSP_HDR/XDES pages, returning an Innodb::Page object
   # for each one.
   def each_xdes_page
@@ -350,9 +393,9 @@ class Innodb::Space
       return enum_for(:each_xdes_page)
     end
 
-    xdes_page_numbers.each do |page_number|
+    each_xdes_page_number do |page_number|
       current_page = page(page_number)
-      yield current_page if current_page
+      yield current_page if current_page and [:FSP_HDR, :XDES].include?(current_page.type)
     end
   end
 

data/lib/innodb/system.rb

@@ -9,6 +9,9 @@ class Innodb::System
   # A hash of spaces by space ID.
   attr_reader :spaces
 
+  # Array of space names for which a space file was not found.
+  attr_reader :orphans
+
   # The Innodb::DataDictionary for this system.
   attr_reader :data_dictionary
 
@@ -17,6 +20,7 @@ class Innodb::System
 
   def initialize(system_space_file)
     @spaces = {}
+    @orphans = []
     @config = {
       :datadir => File.dirname(system_space_file),
     }
@@ -47,10 +51,20 @@ class Innodb::System
     add_space(space)
   end
 
+  # Add an orphaned space.
+  def add_space_orphan(space_file)
+    orphans << space_file
+  end
+
   # Add a space by table name, constructing an appropriate filename
   # from the provided table name.
   def add_table(table_name)
-    add_space_file("%s/%s.ibd" % [config[:datadir], table_name])
+    space_file = "%s/%s.ibd" % [config[:datadir], table_name]
+    if File.exist?(space_file)
+      add_space_file(space_file)
+    else
+      add_space_orphan(table_name)
+    end
   end
 
   # Return an Innodb::Space object for a given space ID, looking up
@@ -93,6 +107,19 @@ class Innodb::System
     nil
   end
 
+  # Iterate throught all orphaned spaces.
+  def each_orphan
+    unless block_given?
+      return enum_for(:each_orphan)
+    end
+
+    orphans.each do |space_name|
+      yield space_name
+    end
+
+    nil
+  end
+
   # Iterate through all column names by table name.
   def each_column_name_by_table_name(table_name)
     unless block_given?
@@ -150,7 +177,12 @@ class Innodb::System
 
   # Return an array of the table name and index name given an index ID.
   def table_and_index_name_by_id(index_id)
-    if index_record = data_dictionary.index_by_id(index_id)
+    if dd_index = data_dictionary.data_dictionary_index_ids[index_id]
+      # This is a data dictionary index, which won't be found in the data
+      # dictionary itself.
+      [dd_index[:table], dd_index[:index]]
+    elsif index_record = data_dictionary.index_by_id(index_id)
+      # This is a system or user index.
       [table_name_by_id(index_record["TABLE_ID"]), index_record["NAME"]]
     end
   end

data/lib/innodb/util/buffer_cursor.rb

@@ -250,49 +250,49 @@ class BufferCursor
   def get_uint8(position=nil)
     seek(position)
     data = read_and_advance(1)
-    BinData::Uint8.read(data)
+    BinData::Uint8.read(data).to_i
   end
 
   # Read a big-endian unsigned 16-bit integer.
   def get_uint16(position=nil)
     seek(position)
     data = read_and_advance(2)
-    BinData::Uint16be.read(data)
+    BinData::Uint16be.read(data).to_i
   end
 
   # Read a big-endian signed 16-bit integer.
   def get_sint16(position=nil)
     seek(position)
     data = read_and_advance(2)
-    BinData::Int16be.read(data)
+    BinData::Int16be.read(data).to_i
   end
 
   # Read a big-endian unsigned 24-bit integer.
   def get_uint24(position=nil)
     seek(position)
     data = read_and_advance(3)
-    BinData::Uint24be.read(data)
+    BinData::Uint24be.read(data).to_i
   end
 
   # Read a big-endian unsigned 32-bit integer.
   def get_uint32(position=nil)
     seek(position)
     data = read_and_advance(4)
-    BinData::Uint32be.read(data)
+    BinData::Uint32be.read(data).to_i
   end
 
   # Read a big-endian unsigned 48-bit integer.
   def get_uint48(position=nil)
     seek(position)
     data = read_and_advance(6)
-    BinData::Uint48be.read(data)
+    BinData::Uint48be.read(data).to_i
   end
 
   # Read a big-endian unsigned 64-bit integer.
   def get_uint64(position=nil)
     seek(position)
     data = read_and_advance(8)
-    BinData::Uint64be.read(data)
+    BinData::Uint64be.read(data).to_i
   end
 
   # Read a big-endian unsigned integer given its size in bytes.

data/lib/innodb/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Innodb
-  VERSION = "0.9.8"
+  VERSION = "0.9.9"
 end

data/lib/innodb/xdes.rb

@@ -117,9 +117,9 @@ class Innodb::Xdes
     bitmap = xdes[:bitmap].enum_for(:each_byte)
 
     bitmap.each_with_index do |byte, byte_index|
-      (0..3).each_with_index do |page, page_index|
-        page_number = xdes[:start_page] + (byte_index * 4) + page_index
-        page_bits = ((byte >> (page * BITS_PER_PAGE)) & BITMAP_BV_ALL)
+      (0..3).each do |page_offset|
+        page_number = xdes[:start_page] + (byte_index * 4) + page_offset
+        page_bits = ((byte >> (page_offset * BITS_PER_PAGE)) & BITMAP_BV_ALL)
         page_status = {
           :free   => (page_bits & BITMAP_BV_FREE  != 0),
           :clean  => (page_bits & BITMAP_BV_CLEAN != 0),

data/lib/innodb.rb

@@ -27,6 +27,7 @@ require "innodb/data_dictionary"
 require "innodb/page"
 require "innodb/page/blob"
 require "innodb/page/fsp_hdr_xdes"
+require "innodb/page/ibuf_bitmap"
 require "innodb/page/inode"
 require "innodb/page/index"
 require "innodb/page/trx_sys"
@@ -38,6 +39,8 @@ require "innodb/space"
 require "innodb/system"
 require "innodb/history"
 require "innodb/history_list"
+require "innodb/ibuf_bitmap"
+require "innodb/ibuf_index"
 require "innodb/inode"
 require "innodb/index"
 require "innodb/log_record"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: innodb_ruby
 version: !ruby/object:Gem::Version
-  version: 0.9.8
+  version: 0.9.9
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-04 00:00:00.000000000 Z
+date: 2014-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bindata
@@ -47,6 +47,8 @@ files:
 - lib/innodb/fseg_entry.rb
 - lib/innodb/history.rb
 - lib/innodb/history_list.rb
+- lib/innodb/ibuf_bitmap.rb
+- lib/innodb/ibuf_index.rb
 - lib/innodb/index.rb
 - lib/innodb/inode.rb
 - lib/innodb/list.rb
@@ -59,11 +61,13 @@ files:
 - lib/innodb/page.rb
 - lib/innodb/page/blob.rb
 - lib/innodb/page/fsp_hdr_xdes.rb
+- lib/innodb/page/ibuf_bitmap.rb
 - lib/innodb/page/index.rb
 - lib/innodb/page/index_compressed.rb
 - lib/innodb/page/inode.rb
 - lib/innodb/page/sys.rb
 - lib/innodb/page/sys_data_dictionary_header.rb
+- lib/innodb/page/sys_ibuf_header.rb
 - lib/innodb/page/sys_rseg_header.rb
 - lib/innodb/page/trx_sys.rb
 - lib/innodb/page/undo_log.rb