innodb_ruby 0.7.3 → 0.7.4

data/bin/innodb_space

@@ -113,13 +113,13 @@ def space_indexes(space)
       used =
         fragments +
         fseg[:not_full_n_used] +
-        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:full].base[:length]
+        space.pages_per_extent * fseg[:full].base[:length]
 
       allocated =
         fragments +
-        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:full].base[:length] +
-        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:not_full].base[:length] +
-        Innodb::Xdes::PAGES_PER_EXTENT * fseg[:free].base[:length]
+        space.pages_per_extent * fseg[:full].base[:length] +
+        space.pages_per_extent * fseg[:not_full].base[:length] +
+        space.pages_per_extent * fseg[:free].base[:length]
 
       puts "%-12i%-12i%-12s%-12i%-12i%-12s" % [
         index.id,
@@ -206,7 +206,7 @@ def space_index_pages_free_plot(space, image)
 
   image_file = image + "_free.png"
   # Aim for one horizontal pixel per extent, but min 1k and max 10k width.
-  image_width = [10000, [1000, space.pages / 64].max].min
+  image_width = [10000, [1000, space.pages / space.pages_per_extent].max].min
 
   Gnuplot.open do |gp|
     Gnuplot::Plot.new(gp) do |plot|
@@ -334,6 +334,10 @@ Usage: innodb_space -f <file> [-p <page>] [-l <level>] <mode> [<mode>, ...]
   --file, -f <file>
     Load the tablespace file <file>.
 
+  --page-size, -P <size>
+    Provide the page size (in KiB): 16 (the default), 8, 4, 2, 1. Page sizes
+    other than 16 may not work well, or at all.
+
   --page, -p <page>
     Operate on the page <page>; may be specified more than once.
 
@@ -411,6 +415,7 @@ end
 
 @options = OpenStruct.new
 @options.file   = nil
+@options.page_size = Innodb::Space::DEFAULT_PAGE_SIZE
 @options.pages  = []
 @options.levels = []
 @options.lists  = []
@@ -419,6 +424,7 @@ end
 getopt_options = [
   [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
   [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--page-size",        "-P",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--page",             "-p",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--level",            "-l",     GetoptLong::REQUIRED_ARGUMENT ],
   [ "--list",             "-L",     GetoptLong::REQUIRED_ARGUMENT ],
@@ -436,6 +442,11 @@ getopt.each do |opt, arg|
     @options.mode = arg
   when "--file"
     @options.file = arg
+  when "--page-size"
+    unless [1, 2, 4, 8, 16].include?(arg.to_i)
+      usage 1, "Page size #{arg} is not understood"
+    end
+    @options.page_size = arg.to_i * 1024
   when "--page"
     @options.pages << arg.to_i
   when "--level"
@@ -453,7 +464,7 @@ unless @options.file
   usage 1, "File must be provided with -f argument"
 end
 
-space = Innodb::Space.new(@options.file)
+space = Innodb::Space.new(@options.file, @options.page_size)
 
 if @options.describer
   space.record_describer = Innodb::RecordDescriber.const_get(@options.describer)

data/lib/innodb/index.rb

@@ -1,10 +1,13 @@
 # An InnoDB index B-tree, given an Innodb::Space and a root page number.
 class Innodb::Index
   attr_reader :root
+  attr_reader :stats
+  attr_accessor :debug
 
   def initialize(space, root_page_number)
     @space = space
     @root = @space.page(root_page_number)
+    @debug = false
 
     unless @root
       raise "Page #{root_page_number} couldn't be read"
@@ -19,6 +22,12 @@ class Innodb::Index
     unless @root.prev.nil? && @root.next.nil?
       raise "Page #{root_page_number} is a node page, but not appear to be the root; it has previous page and next page pointers"
     end
+
+    reset_stats
+  end
+
+  def reset_stats
+    @stats = Hash.new(0)
   end
 
   # A helper function to access the index ID in the page header.
@@ -114,11 +123,14 @@ class Innodb::Index
   #   -1 = a is less than b
   #   +1 = a is greater than b
   def compare_key(a, b)
+    @stats[:compare_key] += 1
+
     return 0 if a.nil? && b.nil?
     return -1 if a.nil? || (!b.nil? && a.size < b.size)
     return +1 if b.nil? || (!a.nil? && a.size > b.size)
 
     a.each_index do |i|
+      @stats[:compare_key_field_comparison] += 1
       return -1 if a[i] < b[i]
       return +1 if a[i] > b[i]
     end
@@ -131,12 +143,30 @@ class Innodb::Index
   # than the key. (If an exact match is desired, compare_key must be used to
   # check if the returned record matches. This makes the function useful for
   # search in both leaf and non-leaf pages.)
-  def linear_search_from_cursor(cursor, key)
+  def linear_search_from_cursor(page, cursor, key)
+    @stats[:linear_search_from_cursor] += 1
+
     this_rec = cursor.record
 
+    if @debug
+      puts "linear_search_from_cursor: start=(%s), page=%i, level=%i" % [
+        this_rec && this_rec[:key].join(", "),
+        page.offset,
+        page.level,
+      ]
+    end
+
     # Iterate through all records until finding either a matching record or
     # one whose key is greater than the desired key.
     while this_rec && next_rec = cursor.record
+      @stats[:linear_search_from_cursor_record_scans] += 1
+
+      if @debug
+        puts "linear_search_from_cursor: scanning: current=(%s)" % [
+          this_rec && this_rec[:key].join(", "),
+        ]
+      end
+
       # If we reach supremum, return the last non-system record we got.
       return this_rec if next_rec[:header][:type] == :supremum
 
@@ -163,8 +193,18 @@ class Innodb::Index
   # desired, the returned record must be checked in the same way as the above
   # linear_search_from_cursor function.)
   def binary_search_by_directory(page, dir, key)
+    @stats[:binary_search_by_directory] += 1
+
     return nil if dir.empty?
 
+    if @debug
+      puts "binary_search_by_directory: page=%i, level=%i, dir.size=%i" % [
+        page.offset,
+        page.level,
+        dir.size,
+      ]
+    end
+
     # Split the directory at the mid-point (using integer math, so the division
     # is rounding down). Retrieve the record that sits at the mid-point.
     mid = dir.size / 2
@@ -175,27 +215,28 @@ class Innodb::Index
     # is the beginning of the page there can't be many records left to check
     # anyway.
     if rec[:header][:type] == :infimum
-      return linear_search_from_cursor(page.record_cursor(rec[:next]), key)
+      return linear_search_from_cursor(page, page.record_cursor(rec[:next]), key)
     end
 
     # Compare the desired key to the mid-point record's key.
     case compare_key(key, rec[:key])
     when 0
       # An exact match for the key was found. Return the record.
+      @stats[:binary_search_by_directory_exact_match] += 1
       rec
     when +1
       # The mid-point record's key is less than the desired key.
       if dir.size == 1
         # This is the last entry remaining from the directory, use linear
-        # search to find the record. We already know that there wasn't an
-        # exact match, so skip the current record and start cursoring from
-        # the next record.
-        linear_search_from_cursor(page.record_cursor(rec[:next]), key)
+        # search to find the record.
+        @stats[:binary_search_by_directory_linear_search] += 1
+        linear_search_from_cursor(page, page.record_cursor(rec[:offset]), key)
       else
         # There are more entries remaining from the directory, recurse again
         # using binary search on the right half of the directory, which
         # represents values greater than or equal to the mid-point record's
         # key.
+        @stats[:binary_search_by_directory_recurse_right] += 1
         binary_search_by_directory(page, dir[mid...dir.size], key)
       end
     when -1
@@ -203,10 +244,12 @@ class Innodb::Index
       if dir.size == 1
         # If this is the last entry remaining from the directory, we didn't
         # find anything workable.
+        @stats[:binary_search_by_directory_empty_result] += 1
         nil
       else
         # Recurse on the left half of the directory, which represents values
         # less than the mid-point record's key.
+        @stats[:binary_search_by_directory_recurse_left] += 1
         binary_search_by_directory(page, dir[0...mid], key)
       end
     end
@@ -218,10 +261,20 @@ class Innodb::Index
   # record is not found, nil is returned (either because linear_search_in_page
   # returns nil breaking the loop, or because compare_key returns non-zero).
   def linear_search(key)
+    @stats[:linear_search] += 1
+
     page = @root
 
+    if @debug
+      puts "linear_search: key=(%s), root=%i, level=%i" % [
+        key.join(", "),
+        page.offset,
+        page.level,
+      ]
+    end
+
     while rec =
-      linear_search_from_cursor(page.record_cursor(page.infimum[:next]), key)
+      linear_search_from_cursor(page, page.record_cursor(page.infimum[:next]), key)
       if page.level > 0
         # If we haven't reached a leaf page yet, move down the tree and search
         # again using linear search.
@@ -240,8 +293,18 @@ class Innodb::Index
   # the page directory to search while making as few record comparisons as
   # possible. If a matching record is not found, nil is returned.
   def binary_search(key)
+    @stats[:binary_search] += 1
+
     page = @root
 
+    if @debug
+      puts "binary_search: key=(%s), root=%i, level=%i" % [
+        key.join(", "),
+        page.offset,
+        page.level,
+      ]
+    end
+
     while rec = binary_search_by_directory(page, page.directory, key)
       if page.level > 0
         # If we haven't reached a leaf page yet, move down the tree and search

data/lib/innodb/page.rb

@@ -32,13 +32,19 @@ class Innodb::Page
     page
   end
 
-  # Initialize a page by passing in a 16kB buffer containing the raw page
-  # contents. Currently only 16kB pages are supported.
+  # Initialize a page by passing in a buffer containing the raw page contents.
+  # The buffer size should match the space's page size.
   def initialize(space, buffer)
+    unless space.page_size == buffer.size
+      raise "Buffer size #{buffer.size} is different than space page size"
+    end
+
     @space  = space
     @buffer = buffer
   end
 
+  attr_reader :space
+
   # Return the page size, to eventually be able to deal with non-16kB pages.
   def size
     @size ||= @buffer.size

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -11,10 +11,6 @@ require "innodb/xdes"
 # The basic structure of FSP_HDR and XDES pages is: FIL header, FSP header,
 # an array of 256 XDES entries, empty (unused) space, and FIL trailer.
 class Innodb::Page::FspHdrXdes < Innodb::Page
-  # This is actually defined as page size divided by extent size, which is
-  # 16384 / 64 = 256.
-  XDES_N_ARRAY_ENTRIES = 256
-
   # The FSP header immediately follows the FIL header.
   def pos_fsp_header
     pos_fil_header + size_fil_header
@@ -31,6 +27,12 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
     pos_fsp_header + size_fsp_header
   end
 
+  # The number of entries in the XDES array. Defined as page size divided by
+  # extent size.
+  def entries_in_xdes_array
+    size / space.pages_per_extent
+  end
+
   # Read the FSP (filespace) header, which contains a few counters and flags,
   # as well as list base nodes for each list maintained in the filespace.
   def fsp_header
@@ -66,7 +68,7 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
     end
 
     c = cursor(pos_xdes_array)
-    XDES_N_ARRAY_ENTRIES.times do
+    entries_in_xdes_array.times do
       yield Innodb::Xdes.new(self, c)
     end
   end

data/lib/innodb/page/inode.rb

@@ -45,7 +45,7 @@ class Innodb::Page::Inode < Innodb::Page
 
   # Return the list entry.
   def list_entry
-    c = cursor(pos_inode_list_entry)
+    c = cursor(pos_list_entry)
     Innodb::List.get_node(c)
   end
 

data/lib/innodb/space.rb

@@ -1,27 +1,49 @@
 # An InnoDB tablespace file, which can be either a multi-table ibdataN file
 # or a single-table "innodb_file_per_table" .ibd file.
 class Innodb::Space
-  attr_accessor :record_describer
-  attr_reader :pages
+  # InnoDB's default page size is 16KiB.
+  DEFAULT_PAGE_SIZE = 16384
 
-  # Currently only 16kB InnoDB pages are supported.
-  PAGE_SIZE = 16384
-
-  # Open a tablespace file.
-  def initialize(file)
+  # Open a tablespace file, providing the page size to use. Pages that aren't
+  # 16 KiB may not be supported well.
+  def initialize(file, page_size=DEFAULT_PAGE_SIZE)
     @file = File.open(file)
+    @page_size = page_size
     @size = @file.stat.size
-    @pages = (@size / PAGE_SIZE)
+    @pages = (@size / page_size)
     @record_describer = nil
   end
 
+  # An object which can be used to describe records found in pages within
+  # this space.
+  attr_accessor :record_describer
+
+  # The size (in bytes) of each page in the space.
+  attr_reader :page_size
+
+  # The size (in bytes) of the space
+  attr_reader :size
+
+  # The number of pages in the space.
+  attr_reader :pages
+
+  # The number of pages per extent.
+  def pages_per_extent
+    64
+  end
+
+  # The size (in bytes) of an extent.
+  def extent_size
+    page_size * pages_per_extent
+  end
+
   # Get an Innodb::Page object for a specific page by page number.
   def page(page_number)
-    offset = page_number.to_i * PAGE_SIZE
+    offset = page_number.to_i * page_size
     return nil unless offset < @size
-    return nil unless (offset + PAGE_SIZE) <= @size
+    return nil unless (offset + page_size) <= @size
     @file.seek(offset)
-    page_data = @file.read(PAGE_SIZE)
+    page_data = @file.read(page_size)
     this_page = Innodb::Page.parse(self, page_data)
 
     if this_page.type == :INDEX
@@ -51,7 +73,8 @@ class Innodb::Space
     end
 
     (3...@pages).each do |page_number|
-      if page(page_number).root?
+      page = page(page_number)
+      if page.type == :INDEX && page.root?
         yield index(page_number)
       else
         break

data/lib/innodb/version.rb

@@ -1,3 +1,3 @@
 module Innodb
-  VERSION = "0.7.3"
+  VERSION = "0.7.4"
 end

data/lib/innodb/xdes.rb

@@ -3,10 +3,6 @@
 #
 # Note the distinction between +XDES+ _entries_ and +XDES+ _pages_.
 class Innodb::Xdes
-  # Number of pages contained in an extent. InnoDB extents are normally
-  # 64 pages, or 1MiB in size.
-  PAGES_PER_EXTENT = 64
-
   # Number of bits per page in the +XDES+ entry bitmap field. Currently
   # +XDES+ entries store two bits per page, with the following meanings:
   #
@@ -23,12 +19,6 @@ class Innodb::Xdes
   # The bitwise-OR of all bitmap bit values.
   BITMAP_BV_ALL = (BITMAP_BV_FREE | BITMAP_BV_CLEAN)
 
-  # Size (in bytes) of the bitmap field in the +XDES+ entry.
-  BITMAP_SIZE = (PAGES_PER_EXTENT * BITS_PER_PAGE) / 8
-
-  # Size (in bytes) of the an +XDES+ entry.
-  ENTRY_SIZE = 8 + Innodb::List::NODE_SIZE + 4 + BITMAP_SIZE
-
   # The values used in the +:state+ field indicating what the extent is
   # used for (or what list it is on).
   STATES = {
@@ -51,15 +41,30 @@ class Innodb::Xdes
 
   def initialize(page, cursor)
     @page = page
-    extent_number = (cursor.position - page.pos_xdes_array) / ENTRY_SIZE
-    start_page = page.offset + (extent_number * PAGES_PER_EXTENT)
-    @xdes = {
+    @xdes = read_xdes_entry(page, cursor)
+  end
+
+  # Size (in bytes) of the bitmap field in the +XDES+ entry.
+  def size_bitmap
+    (@page.space.pages_per_extent * BITS_PER_PAGE) / 8
+  end
+
+  # Size (in bytes) of the an +XDES+ entry.
+  def size_entry
+    8 + Innodb::List::NODE_SIZE + 4 + size_bitmap
+  end
+
+  # Read an XDES entry from a cursor.
+  def read_xdes_entry(page, cursor)
+    extent_number = (cursor.position - page.pos_xdes_array) / size_entry
+    start_page = page.offset + (extent_number * page.space.pages_per_extent)
+    {
       :start_page => start_page,
       :fseg_id    => cursor.get_uint64,
       :this       => {:page => page.offset, :offset => cursor.position},
       :list       => Innodb::List.get_node(cursor),
       :state      => STATES[cursor.get_uint32],
-      :bitmap     => cursor.get_bytes(BITMAP_SIZE),
+      :bitmap     => cursor.get_bytes(size_bitmap),
     }
   end
 
@@ -104,7 +109,7 @@ class Innodb::Xdes
 
   # Return the count of used pages (free bit is false) on this extent.
   def used_pages
-    PAGES_PER_EXTENT - free_pages
+    @page.space.pages_per_extent - free_pages
   end
 
   # Return the address of the previous list pointer from the list node

metadata

@@ -1,35 +1,48 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: innodb_ruby
-version: !ruby/object:Gem::Version
-  version: 0.7.3
+version: !ruby/object:Gem::Version 
+  hash: 11
   prerelease: 
+  segments: 
+  - 0
+  - 7
+  - 4
+  version: 0.7.4
 platform: ruby
-authors:
+authors: 
 - Jeremy Cole
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-16 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-12-20 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: bindata
-  requirement: &70099335724160 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 4
+        - 5
         version: 1.4.5
   type: :runtime
-  prerelease: false
-  version_requirements: *70099335724160
+  version_requirements: *id001
 description: Library for parsing InnoDB data files in Ruby
 email: jeremy@jcole.us
-executables:
+executables: 
 - innodb_log
 - innodb_space
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - README.md
 - lib/innodb.rb
 - lib/innodb/cursor.rb
@@ -52,27 +65,37 @@ files:
 - bin/innodb_space
 homepage: http://jcole.us/
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: InnoDB data file parser
 test_files: []
+
 has_rdoc: