innodb_ruby 0.7.7 → 0.7.10

data/bin/innodb_space

@@ -229,6 +229,32 @@ def space_extents(space)
   print_xdes_list(space.each_xdes)
 end
 
+def page_directory_summary(page)
+  puts "%-8s%-8s%-14s%-8s%s" % [
+    "slot",
+    "offset",
+    "type",
+    "owned",
+    "key",
+  ]
+
+  page.directory.each_with_index do |offset, slot|
+    record = page.record(offset)
+    key = if [:conventional, :node_pointer].include? record[:header][:type]
+      if record[:key]
+        "(%s)" % record[:key].join(", ")
+      end
+    end
+    puts "%-8i%-8i%-14s%-8i%s" % [
+      slot,
+      offset,
+      record[:header][:type],
+      record[:header][:n_owned],
+      key,
+    ]
+  end
+end
+
 def index_fseg_lists(index, fseg_name)
   unless index.fseg(fseg_name)
     raise "File segment '#{fseg_name}' doesn't exist"
@@ -362,8 +388,8 @@ Usage: innodb_space -f <file> [-p <page>] [-l <level>] <mode> [<mode>, ...]
     Load the tablespace file <file>.
 
   --page-size, -P <size>
-    Provide the page size (in KiB): 16 (the default), 8, 4, 2, 1. Page sizes
-    other than 16 may not work well, or at all.
+    Provide the page size, overriding auto-detection (in KiB): 16, 8, 4, 2, 1.
+    Page sizes other than 16 may not work well, or at all.
 
   --page, -p <page>
     Operate on the page <page>; may be specified more than once.
@@ -383,9 +409,6 @@ Usage: innodb_space -f <file> [-p <page>] [-l <level>] <mode> [<mode>, ...]
 
 The following modes are supported:
 
-  page-dump
-    Dump the contents of the page, using the Ruby pp ("pretty-print") module.
-
   space-summary
     Summarize all pages within a tablespace. A starting page number can be
     provided with the --page/-p argument.
@@ -421,6 +444,13 @@ The following modes are supported:
   space-extents
     Iterate through all extents, printing the extent descriptor bitmap.
 
+  page-dump
+    Dump the contents of a page, using the Ruby pp ("pretty-print") module.
+
+  page-directory-summary
+    Summarize the record contents of the page directory in a page. If a record
+    describer is available, the key of each record will be printed.
+
   index-recurse
     Recurse an index, starting at the root (which must be provided in the first
     --page/-p argument), printing the node pages, node pointers (links), leaf
@@ -457,9 +487,12 @@ END_OF_USAGE
   exit exit_code
 end
 
+Signal.trap("INT")  { exit }
+Signal.trap("PIPE") { exit }
+
 @options = OpenStruct.new
 @options.file   = nil
-@options.page_size = Innodb::Space::DEFAULT_PAGE_SIZE
+@options.page_size = nil
 @options.pages  = []
 @options.levels = []
 @options.lists  = []
@@ -525,9 +558,20 @@ ARGV.each do |mode|
       usage 1, "Page numbers to dump must be provided with --page/-p"
     end
 
-    @options.pages.each do |page|
-      space.page(page).dump
+    @options.pages.each do |page_number|
+      space.page(page_number).dump
     end
+  when "page-directory-summary"
+    if @options.pages.empty?
+      usage 1, "Page numbers to dump must be provided with --page/-p"
+    end
+
+    page = space.page(@options.pages.first)
+    if page.type != :INDEX
+      usage 1, "Page must be an index page"
+    end
+
+    page_directory_summary(page)
   when "space-summary"
     space_summary(space, @options.pages.first || 0)
   when "space-index-pages-summary"

data/lib/innodb/index.rb

@@ -187,10 +187,10 @@ class Innodb::Index
     this_rec = cursor.record
 
     if @debug
-      puts "linear_search_from_cursor: start=(%s), page=%i, level=%i" % [
-        this_rec && this_rec[:key].join(", "),
+      puts "linear_search_from_cursor: page=%i, level=%i, start=(%s)" % [
         page.offset,
         page.level,
+        this_rec && this_rec[:key].join(", "),
       ]
     end
 
@@ -200,7 +200,9 @@ class Innodb::Index
       @stats[:linear_search_from_cursor_record_scans] += 1
 
       if @debug
-        puts "linear_search_from_cursor: scanning: current=(%s)" % [
+        puts "linear_search_from_cursor: page=%i, level=%i, current=(%s)" % [
+          page.offset,
+          page.level,
           this_rec && this_rec[:key].join(", "),
         ]
       end
@@ -208,6 +210,10 @@ class Innodb::Index
       # If we reach supremum, return the last non-system record we got.
       return this_rec if next_rec[:header][:type] == :supremum
 
+      if compare_key(key, this_rec[:key]) < 0
+        return this_rec
+      end
+
       if (compare_key(key, this_rec[:key]) >= 0) &&
         (compare_key(key, next_rec[:key]) < 0)
         # The desired key is either an exact match for this_rec or is greater
@@ -235,19 +241,21 @@ class Innodb::Index
 
     return nil if dir.empty?
 
+    # Split the directory at the mid-point (using integer math, so the division
+    # is rounding down). Retrieve the record that sits at the mid-point.
+    mid = ((dir.size-1) / 2)
+    rec = page.record(dir[mid])
+
     if @debug
-      puts "binary_search_by_directory: page=%i, level=%i, dir.size=%i" % [
+      puts "binary_search_by_directory: page=%i, level=%i, dir.size=%i, dir[%i]=(%s)" % [
         page.offset,
         page.level,
         dir.size,
+        mid,
+        rec[:key] && rec[:key].join(", "),
       ]
     end
 
-    # Split the directory at the mid-point (using integer math, so the division
-    # is rounding down). Retrieve the record that sits at the mid-point.
-    mid = dir.size / 2
-    rec = page.record(dir[mid])
-
     # The mid-point record was the infimum record, which is not comparable with
     # compare_key, so we need to just linear scan from here. If the mid-point
     # is the beginning of the page there can't be many records left to check
@@ -264,18 +272,29 @@ class Innodb::Index
       rec
     when +1
       # The mid-point record's key is less than the desired key.
-      if dir.size == 1
-        # This is the last entry remaining from the directory, use linear
-        # search to find the record.
-        @stats[:binary_search_by_directory_linear_search] += 1
-        linear_search_from_cursor(page, page.record_cursor(rec[:offset]), key)
-      else
+      if dir.size > 2
         # There are more entries remaining from the directory, recurse again
         # using binary search on the right half of the directory, which
         # represents values greater than or equal to the mid-point record's
         # key.
         @stats[:binary_search_by_directory_recurse_right] += 1
         binary_search_by_directory(page, dir[mid...dir.size], key)
+      else
+        next_rec = page.record(dir[mid+1])
+        next_key = next_rec && compare_key(key, next_rec[:key])
+        if dir.size == 1 || next_key == -1 || next_key == 0
+          # This is the last entry remaining from the directory, or our key is
+          # greater than rec and less than rec+1's key. Use linear search to
+          # find the record starting at rec.
+          @stats[:binary_search_by_directory_linear_search] += 1
+          linear_search_from_cursor(page, page.record_cursor(rec[:offset]), key)
+        elsif next_key == +1
+          puts "+1"
+          @stats[:binary_search_by_directory_linear_search] += 1
+          linear_search_from_cursor(page, page.record_cursor(next_rec[:offset]), key)
+        else
+          nil
+        end
       end
     when -1
       # The mid-point record's key is greater than the desired key.
@@ -304,10 +323,10 @@ class Innodb::Index
     page = @root
 
     if @debug
-      puts "linear_search: key=(%s), root=%i, level=%i" % [
-        key.join(", "),
+      puts "linear_search: root=%i, level=%i, key=(%s)" % [
         page.offset,
         page.level,
+        key.join(", "),
       ]
     end
 
@@ -336,14 +355,16 @@ class Innodb::Index
     page = @root
 
     if @debug
-      puts "binary_search: key=(%s), root=%i, level=%i" % [
-        key.join(", "),
+      puts "binary_search: root=%i, level=%i, key=(%s)" % [
         page.offset,
         page.level,
+        key.join(", "),
       ]
     end
 
-    while rec = binary_search_by_directory(page, page.directory, key)
+    # Remove supremum from the page directory, since nothing can be scanned
+    # linearly from there anyway.
+    while rec = binary_search_by_directory(page, page.directory[0...-1], key)
       if page.level > 0
         # If we haven't reached a leaf page yet, move down the tree and search
         # again using binary search.

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -11,6 +11,45 @@ require "innodb/xdes"
 # The basic structure of FSP_HDR and XDES pages is: FIL header, FSP header,
 # an array of 256 XDES entries, empty (unused) space, and FIL trailer.
 class Innodb::Page::FspHdrXdes < Innodb::Page
+  # A value added to the adjusted exponent stored in the page size field of
+  # the flags in the FSP header.
+  FLAGS_PAGE_SIZE_ADJUST = 9
+
+  # Read a given number of bits from an integer at a specific bit offset. The
+  # value returned is 0-based so does not need further shifting or adjustment.
+  def self.read_bits_at_offset(data, bits, offset)
+    ((data & (((1 << bits) - 1) << offset)) >> offset)
+  end
+
+  # Decode the "flags" field in the FSP header, returning a hash of useful
+  # decoded flags. Unfortunately, InnoDB has a fairly weird and broken
+  # implementation of these flags. The flags are:
+  #
+  # Offset    Size    Description
+  # 0         1       Page Format (redundant, compact). This is unfortunately
+  #                   coerced to 0 if it is "compact" and no other flags are
+  #                   set, making it useless to innodb_ruby.
+  # 1         4       Compressed Page Size (zip_size). This is stored as a
+  #                   power of 2, minus 9. Since 0 is reserved to mean "not
+  #                   compressed", the minimum value is 1, thus making the
+  #                   smallest page size 1024 (2 ** (9 + 1)).
+  # 5         1       Table Format (Antelope, Barracuda). This was supposed
+  #                   to reserve 6 bits, but due to a bug in InnoDB only
+  #                   actually reserved 1 bit.
+  #
+  def self.decode_flags(flags)
+    # The page size for compressed pages is stored at bit offset 1 and consumes
+    # 4 bits. Value 0 means the page is not compressed.
+    page_size = read_bits_at_offset(flags, 4, 1)
+    {
+      :compressed => page_size == 0 ? false : true,
+      :page_size => page_size == 0 ?
+        Innodb::Space::DEFAULT_PAGE_SIZE :
+        (1 << (FLAGS_PAGE_SIZE_ADJUST + page_size)),
+      :value => flags,
+    }
+  end
+
   # The FSP header immediately follows the FIL header.
   def pos_fsp_header
     pos_fil_header + size_fil_header
@@ -42,7 +81,7 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
       :unused             => c.get_uint32,
       :size               => c.get_uint32,
       :free_limit         => c.get_uint32,
-      :flags              => c.get_uint32,
+      :flags              => self.class.decode_flags(c.get_uint32),
       :frag_n_used        => c.get_uint32,
       :free               => Innodb::List::Xdes.new(@space,
                               Innodb::List.get_base_node(c)),

data/lib/innodb/space.rb

@@ -4,13 +4,20 @@ class Innodb::Space
   # InnoDB's default page size is 16KiB.
   DEFAULT_PAGE_SIZE = 16384
 
-  # Open a tablespace file, providing the page size to use. Pages that aren't
-  # 16 KiB may not be supported well.
-  def initialize(file, page_size=DEFAULT_PAGE_SIZE)
+  # Open a tablespace file, optionally providing the page size to use. Pages
+  # that aren't 16 KiB may not be supported well.
+  def initialize(file, page_size=nil)
     @file = File.open(file)
-    @page_size = page_size
     @size = @file.stat.size
-    @pages = (@size / page_size)
+
+    if page_size
+      @page_size = page_size
+    else
+      @page_size = fsp_flags[:page_size]
+    end
+
+    @pages = (@size / @page_size)
+    @compressed = fsp_flags[:compressed]
     @record_describer = nil
   end
 
@@ -27,6 +34,42 @@ class Innodb::Space
   # The number of pages in the space.
   attr_reader :pages
 
+  # Read the FSP header "flags" field by byte offset within the space file.
+  # This is useful in order to initialize the page size, as we can't properly
+  # read the FSP_HDR page before we know its size.
+  def raw_fsp_header_flags
+    # A simple sanity check. The FIL header should be initialized in page 0,
+    # to offset 0 and page type :FSP_HDR (8).
+    page_offset = BinData::Uint32be.read(read_at_offset(4, 4))
+    page_type   = BinData::Uint16be.read(read_at_offset(24, 2))
+    unless page_offset == 0 && Innodb::Page::PAGE_TYPE[page_type] == :FSP_HDR
+      raise "Something is very wrong; Page 0 does not seem to be type FSP_HDR"
+    end
+
+    # Another sanity check. The Space ID should be the same in both the FIL
+    # and FSP headers.
+    fil_space = BinData::Uint32be.read(read_at_offset(34, 4))
+    fsp_space = BinData::Uint32be.read(read_at_offset(38, 4))
+    unless fil_space == fsp_space
+      raise "Something is very wrong; FIL and FSP header Space IDs don't match"
+    end
+
+    # Well, we're as sure as we can be. Read the flags field and decode it.
+    flags_value = BinData::Uint32be.read(read_at_offset(54, 4))
+    Innodb::Page::FspHdrXdes.decode_flags(flags_value)
+  end
+
+  # The FSP header flags, decoded. If the page size has not been initialized,
+  # reach into the raw bytes of the FSP_HDR page and attempt to decode the
+  # flags field that way.
+  def fsp_flags
+    if @page_size
+      return fsp[:flags]
+    else
+      raw_fsp_header_flags
+    end
+  end
+
   # The size (in bytes) of an extent.
   def extent_size
     1048576
@@ -48,14 +91,23 @@ class Innodb::Space
     (0..(@pages / pages_per_xdes_page)).map { |n| n * pages_per_xdes_page }
   end
 
-  # Get an Innodb::Page object for a specific page by page number.
-  def page(page_number)
+  # Get the raw byte buffer of size bytes at offset in the file.
+  def read_at_offset(offset, size)
+    @file.seek(offset)
+    @file.read(size)
+  end
+
+  # Get the raw byte buffer for a specific page by page number.
+  def page_data(page_number)
     offset = page_number.to_i * page_size
     return nil unless offset < @size
     return nil unless (offset + page_size) <= @size
-    @file.seek(offset)
-    page_data = @file.read(page_size)
-    this_page = Innodb::Page.parse(self, page_data)
+    read_at_offset(offset, page_size)
+  end
+
+  # Get an Innodb::Page object for a specific page by page number.
+  def page(page_number)
+    this_page = Innodb::Page.parse(self, page_data(page_number))
 
     if this_page.type == :INDEX
       this_page.record_describer = @record_describer
@@ -64,9 +116,14 @@ class Innodb::Space
     this_page
   end
 
+  # Get (and cache) the FSP header from the FSP_HDR page.
+  def fsp
+    @fsp ||= page(0).fsp_header
+  end
+
   # Get an Innodb::List object for a specific list by list name.
   def list(name)
-    page(0).fsp_header[name]
+    fsp[name]
   end
 
   # Get an Innodb::Index object for a specific index by root page number.

data/lib/innodb/version.rb

@@ -1,3 +1,3 @@
 module Innodb
-  VERSION = "0.7.7"
+  VERSION = "0.7.10"
 end

metadata

@@ -1,48 +1,35 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: innodb_ruby
-version: !ruby/object:Gem::Version 
-  hash: 13
+version: !ruby/object:Gem::Version
+  version: 0.7.10
   prerelease: 
-  segments: 
-  - 0
-  - 7
-  - 7
-  version: 0.7.7
 platform: ruby
-authors: 
+authors:
 - Jeremy Cole
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-01-09 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-01-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: bindata
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70135906385360 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 13
-        segments: 
-        - 1
-        - 4
-        - 5
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.4.5
   type: :runtime
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: *70135906385360
 description: Library for parsing InnoDB data files in Ruby
 email: jeremy@jcole.us
-executables: 
+executables:
 - innodb_log
 - innodb_space
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - README.md
 - lib/innodb.rb
 - lib/innodb/cursor.rb
@@ -65,37 +52,27 @@ files:
 - bin/innodb_space
 homepage: http://jcole.us/
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
 summary: InnoDB data file parser
 test_files: []
-
 has_rdoc: