innodb_ruby 0.4 → 0.5.0

data/bin/innodb_dump_space

@@ -3,31 +3,187 @@
 require "pp"
 require "innodb"
 
-file, page_number = ARGV.shift(2)
+class EdgesRecordFormatter
+  def format(page)
+    bytes_per_record = (page.record_space / page.page_header[:n_recs])
+    case
+    when [48, 26].include?(bytes_per_record) # Clustered Key
+      {
+        # PRIMARY KEY (source_id, state, position)
+        :type => :clustered,
+        :key => [
+          [:get_uint64],    # source_id
+          [:get_i_sint8],   # state
+          [:get_i_sint64],  # position
+        ],
+        :row => [
+          [:get_uint32],    # updated_at
+          [:get_uint64],    # destination_id
+          [:get_uint8],     # count
+        ],
+      }
+    when [30, 34].include?(bytes_per_record) # Secondary Key
+      {
+        # INDEX (source_id, destination_id)
+        :type => :secondary,
+        :key => [
+          [:get_uint64],    # source_id
+          [:get_uint64],    # destination_id
+        ],
+        # PKV ([source_id], state, position)
+        :row => [
+          [:get_i_sint8],   # state
+          [:get_i_sint64],  # position
+        ],
+      }
+    end
+  end
+end
 
-space = Innodb::Space.new(file)
-#space.page(page_number).dump
-#exit
-
-page_types = Hash.new(0)
-page_levels = Hash.new(0)
-records_per_level = {}
-puts "%-8s%-8s%-8s%-8s" % ["page", "index", "level", "data"]
-space.each_page do |page_number, page|
-  page_types[page.type] += 1
-  if page.type == :INDEX
-    puts "%-8i%-8i%-8i%-8i" % [
-      page_number,
+class PageSplitTestFormatter
+  def format(page)
+      {
+      # PRIMARY KEY (id)
+      :type => :clustered,
+      :key => [
+        [:get_uint64],  # id
+      ],
+      :row => [],
+    }
+  end
+end
+
+def digraph_index(index)
+  puts "digraph btree {"
+  puts "  rankdir = LR;"
+  puts "  ranksep = 2.0;"
+  index.recurse(
+    lambda do |page, depth|
+      if true #page.level != 0
+        label = "<page>Page %i|(%i records)" % [
+          page.offset,
+          page.page_header[:n_recs],
+        ]
+        page.each_child_page do |child_page_number, child_key|
+          label += "|<dir_%i>(%s)" % [
+            child_page_number,
+            child_key.join(", "),
+          ]
+        end
+        puts "  %spage_%i [ shape = \"record\"; label = \"%s\"; ];" % [
+          "  " * depth,
+          page.offset,
+          label,
+        ]
+      end
+    end,
+    lambda do |parent_page, child_page, child_key, depth|
+      if true #child_page.level > 0
+        puts "  %spage_%i:dir_%i -> page_%i:page:nw;" % [
+          "  " * depth,
+          parent_page.offset,
+          child_page.offset,
+          child_page.offset,
+        ]
+      end
+    end
+  )
+  puts "}"
+end
+
+def recurse_index(index)
+  index.recurse(
+    lambda do |page, depth|
+      puts "%s%s %i: %i records, %i bytes" % [
+        "  " * depth,
+        page.level == 0 ? "LEAF" : "NODE",
+        page.offset,
+        page.page_header[:n_recs],
+        page.record_space,
+      ]
+      if page.level == 0
+        page.each_record do |record|
+          puts "%sRECORD: (%s) -> (%s)" % [
+            "  " * (depth+1),
+            record[:key].join(", "),
+            record[:row].join(", "),
+          ]
+        end
+      end
+    end,
+    lambda do |parent_page, child_page, child_min_key, depth|
+      puts "%sLINK %i -> %i: %s records >= (%s)" % [
+        "  " * depth,
+        parent_page.offset,
+        child_page.offset,
+        child_page.page_header[:n_recs],
+        child_min_key.join(", "),
+      ]
+    end
+  )
+end
+
+def dump_space(space)
+  puts "%-8s%-8s%-8s%-8s%-8s%-8s" % [
+    "page",
+    "index",
+    "level",
+    "data",
+    "free",
+    "records",
+  ]
+
+  space.each_page do |page_number, page|
+    case page.type
+    when :INDEX
+      puts "%-8i%-8i%-8i%-8i%-8i%-8i" % [
+        page_number,
+        page.ph[:index_id],
+        page.ph[:level],
+        page.record_space,
+        page.free_space,
+        page.ph[:n_recs],
+      ]
+    when :ALLOCATED
+      puts "%-8i%-8i%-8i%-8i%-8i%-8i" % [ page_number, 0, 0, 0, Innodb::Space::PAGE_SIZE, 0 ]
+    end
+  end
+end
+
+def dump_index_level(index, level)
+  puts "%-8s%-8s%-8s%-8s%-8s%-8s%-8s" % [
+    "page",
+    "index",
+    "level",
+    "data",
+    "free",
+    "records",
+    "min_key",
+  ]
+
+  index.each_page_at_level(level) do |page|
+    puts "%-8i%-8i%-8i%-8i%-8i%-8i%s" % [
+      page.offset,
       page.ph[:index_id],
       page.ph[:level],
       page.record_space,
+      page.free_space,
+      page.ph[:n_recs],
+      page.first_record[:key].join("|"),
     ]
-    #page_levels[page.ph[:index_id]] ||= Hash.new(0)
-    #page_levels[page.ph[:index_id]][page.ph[:level]] += 1
-    #records_per_level[page.ph[:level]] ||= Hash.new(0)
-    #records_per_level[page.ph[:level]][page.ph[:n_recs]] += 1
-  end
-  if page.type == :ALLOCATED
-    puts "%-8i%-8i%-8i%-8i" % [ page_number, 0, 0, 0 ]
   end
 end
+
+file, page_number = ARGV.shift(2)
+
+formatter = EdgesRecordFormatter.new
+#formatter = PageSplitTestFormatter.new
+space = Innodb::Space.new(file)
+space.record_formatter = formatter
+space.page(page_number).dump
+#space.each_page { |page_number, page| puts "%6i%20s" % [page_number, page.type] }
+#dump_space(space)
+#dump_index_level(Innodb::Index.new(space, page_number), 1)
+#recurse_index(Innodb::Index.new(space, page_number))
+#digraph_index(Innodb::Index.new(space, page_number))
+#Innodb::Index.new(space, page_number).each_record { |record| puts "(#{record[:key].join(", ")}) -> (#{record[:row].join(", ")})" }

data/lib/innodb.rb

@@ -3,6 +3,10 @@ module Innodb; end
 
 require "innodb/version"
 require "innodb/page"
+require "innodb/page/fsp_hdr_xdes"
+require "innodb/page/inode"
+require "innodb/page/index"
 require "innodb/space"
+require "innodb/index"
 require "innodb/log_block"
 require "innodb/log"

data/lib/innodb/cursor.rb

@@ -82,6 +82,11 @@ class Innodb::Cursor
     read_and_advance(length)
   end
 
+  # Return raw bytes as hex.
+  def get_hex(length)
+    read_and_advance(length).bytes.map { |c| "%02x" % c }.join
+  end
+
   # Return a big-endian unsigned 8-bit integer.
   def get_uint8(offset=nil)
     seek(offset)
@@ -145,4 +150,16 @@ class Innodb::Cursor
       raise "Invalid flag #{flag.to_s(16)} seen"
     end
   end
+
+  # Return an InnoDB-munged signed 8-bit integer. (This is only implemented
+  # for positive integers at the moment.)
+  def get_i_sint8
+    get_uint8 ^ (1 << 7)
+  end
+
+  # Return an InnoDB-munged signed 64-bit integer. (This is only implemented
+  # for positive integers at the moment.)
+  def get_i_sint64
+    get_uint64 ^ (1 << 63)
+  end
 end

data/lib/innodb/page.rb

@@ -1,36 +1,33 @@
 require "innodb/cursor"
 
 class Innodb::Page
-  # Currently only 16kB InnoDB pages are supported.
-  PAGE_SIZE = 16384
+  SPECIALIZED_CLASSES = {}
 
-  # InnoDB Page Type constants from include/fil0fil.h.
-  PAGE_TYPE = {
-    0     => :ALLOCATED,      # Freshly allocated page
-    2     => :UNDO_LOG,       # Undo log page
-    3     => :INODE,          # Index node
-    4     => :IBUF_FREE_LIST, # Insert buffer free list
-    5     => :IBUF_BITMAP,    # Insert buffer bitmap
-    6     => :SYS,            # System page
-    7     => :TRX_SYS,        # Transaction system data
-    8     => :FSP_HDR,        # File space header
-    9     => :XDES,           # Extent descriptor page
-    10    => :BLOB,           # Uncompressed BLOB page
-    11    => :ZBLOB,          # First compressed BLOB page
-    12    => :ZBLOB2,         # Subsequent compressed BLOB page
-    17855 => :INDEX,          # B-tree node
-  }
+  # Load a page as a generic page in order to make the "fil" header accessible,
+  # and then attempt to hand off the page to a specialized class to be
+  # re-parsed if possible. If there is no specialized class for this type
+  # of page, return the generic object.
+  def self.parse(buffer)
+    page = Innodb::Page.new(buffer)
+
+    if specialized_class = SPECIALIZED_CLASSES[page.type]
+      page = specialized_class.new(buffer)
+    end
+
+    page
+  end
 
   # Initialize a page by passing in a 16kB buffer containing the raw page
   # contents. Currently only 16kB pages are supported.
   def initialize(buffer)
-    unless buffer.size == PAGE_SIZE
-      raise "Page buffer provided was not #{PAGE_SIZE} bytes" 
-    end
-
     @buffer = buffer
   end
 
+  # Return the page size, to eventually be able to deal with non-16kB pages.
+  def size
+    @size ||= @buffer.size
+  end
+
   # A helper function to return bytes from the page buffer based on offset
   # and length, both in bytes.
   def data(offset, length)
@@ -42,23 +39,59 @@ class Innodb::Page
     Innodb::Cursor.new(self, offset)
   end
 
-  FIL_HEADER_SIZE   = 38
-  FIL_HEADER_START  = 0
+  # Return the byte offset of the start of the "fil" header, which is at the
+  # beginning of the page. Included here primarily for completeness.
+  def pos_fil_header
+    0
+  end
+
+  # Return the size of the "fil" header, in bytes.
+  def size_fil_header
+    38
+  end
+
+  # Return the byte offset of the start of the "fil" trailer, which is at
+  # the end of the page.
+  def pos_fil_trailer
+    size - size_fil_trailer
+  end
+
+  # Return the size of the "fil" trailer, in bytes.
+  def size_fil_trailer
+    8
+  end
+
+  # InnoDB Page Type constants from include/fil0fil.h.
+  PAGE_TYPE = {
+    0     => :ALLOCATED,      # Freshly allocated page
+    2     => :UNDO_LOG,       # Undo log page
+    3     => :INODE,          # Index node
+    4     => :IBUF_FREE_LIST, # Insert buffer free list
+    5     => :IBUF_BITMAP,    # Insert buffer bitmap
+    6     => :SYS,            # System page
+    7     => :TRX_SYS,        # Transaction system data
+    8     => :FSP_HDR,        # File space header
+    9     => :XDES,           # Extent descriptor page
+    10    => :BLOB,           # Uncompressed BLOB page
+    11    => :ZBLOB,          # First compressed BLOB page
+    12    => :ZBLOB2,         # Subsequent compressed BLOB page
+    17855 => :INDEX,          # B-tree node
+  }
 
   # A helper to convert "undefined" values stored in previous and next pointers
   # in the page header to nil.
-  def maybe_undefined(value)
+  def self.maybe_undefined(value)
     value == 4294967295 ? nil : value
   end
 
   # Return the "fil" header from the page, which is common for all page types.
   def fil_header
-    c = cursor(FIL_HEADER_START)
+    c = cursor(pos_fil_header)
     @fil_header ||= {
       :checksum   => c.get_uint32,
       :offset     => c.get_uint32,
-      :prev       => maybe_undefined(c.get_uint32),
-      :next       => maybe_undefined(c.get_uint32),
+      :prev       => Innodb::Page.maybe_undefined(c.get_uint32),
+      :next       => Innodb::Page.maybe_undefined(c.get_uint32),
       :lsn        => c.get_uint64,
       :type       => PAGE_TYPE[c.get_uint16],
       :flush_lsn  => c.get_uint64,
@@ -73,6 +106,12 @@ class Innodb::Page
     fil_header[:type]
   end
 
+  # A helper function to return the page offset from the "fil" header, for
+  # easier access.
+  def offset
+    fil_header[:offset]
+  end
+
   # A helper function to return the page number of the logical previous page
   # (from the doubly-linked list from page to page) from the "fil" header,
   # for easier access.
@@ -87,227 +126,13 @@ class Innodb::Page
     fil_header[:next]
   end
 
-  PAGE_HEADER_SIZE  = 36
-  PAGE_HEADER_START = FIL_HEADER_START + FIL_HEADER_SIZE
-
-  PAGE_TRAILER_SIZE  = 16
-  PAGE_TRAILER_START = PAGE_SIZE - PAGE_TRAILER_SIZE
-
-  FSEG_HEADER_SIZE  = 10
-  FSEG_HEADER_START = PAGE_HEADER_START + PAGE_HEADER_SIZE
-  FSEG_HEADER_COUNT = 2
-
-  MUM_RECORD_SIZE   = 8
-  
-  RECORD_BITS_SIZE  = 3
-  RECORD_NEXT_SIZE  = 2
-
-  # Page direction values possible in the page_header[:direction] field.
-  PAGE_DIRECTION = {
-    1 => :left,
-    2 => :right,
-    3 => :same_rec,
-    4 => :same_page,
-    5 => :no_direction,
-  }
-
-  # Return the size of the header for each record.
-  def size_record_header
-    case page_header[:format]
-    when :compact
-      RECORD_BITS_SIZE + RECORD_NEXT_SIZE
-    when :redundant
-      RECORD_BITS_SIZE + RECORD_NEXT_SIZE + 1
-    end
-  end
-
-  # Return the size of a field in the record header for which no description
-  # could be found (but must be skipped anyway).
-  def size_record_undefined
-    case page_header[:format]
-    when :compact
-      0
-    when :redundant
-      1
-    end
-  end
-
-  # Return the "page" header; currently only "INDEX" pages are supported.
-  def page_header
-    return nil unless type == :INDEX
-
-    c = cursor(PAGE_HEADER_START)
-    @page_header ||= {
-      :n_dir_slots  => c.get_uint16,
-      :heap_top     => c.get_uint16,
-      :n_heap       => ((n_heap = c.get_uint16) & (2**15-1)),
-      :free         => c.get_uint16,
-      :garbage      => c.get_uint16,
-      :last_insert  => c.get_uint16,
-      :direction    => PAGE_DIRECTION[c.get_uint16],
-      :n_direction  => c.get_uint16,
-      :n_recs       => c.get_uint16,
-      :max_trx_id   => c.get_uint64,
-      :level        => c.get_uint16,
-      :index_id     => c.get_uint64,
-      :format       => (n_heap & 1<<15) == 0 ? :redundant : :compact,
-    }
-  end
-  alias :ph :page_header
-
-  # Parse and return simple fixed-format system records, such as InnoDB's
-  # internal infimum and supremum records.
-  def system_record(offset)
-    return nil unless type == :INDEX
-
-    c = cursor(offset)
-    c.adjust(-2)
-    {
-      :next => offset + c.get_sint16,
-      :data => c.get_bytes(8),
-    }
-  end
-
-  # Return the byte offset of the start of the "origin" of the infimum record,
-  # which is always the first record in the singly-linked record chain on any
-  # page, and represents a record with a "lower value than any possible user
-  # record". The infimum record immediately follows the page header.
-  def pos_infimum
-    pos_records + size_record_header + size_record_undefined
-  end
-
-  # Return the infimum record on a page.
-  def infimum
-    @infimum ||= system_record(pos_infimum)
-  end
-
-  # Return the byte offset of the start of the "origin" of the supremum record,
-  # which is always the last record in the singly-linked record chain on any
-  # page, and represents a record with a "higher value than any possible user
-  # record". The supremum record immediately follows the infimum record.
-  def pos_supremum
-    pos_infimum + size_record_header + size_record_undefined + MUM_RECORD_SIZE
-  end
-
-  # Return the supremum record on a page.
-  def supremum
-    @supremum ||= system_record(pos_supremum)
-  end
-
-  # Return the byte offset of the start of records within the page (the
-  # position immediately after the page header).
-  def pos_records
-    FIL_HEADER_SIZE + 
-      PAGE_HEADER_SIZE + 
-      (FSEG_HEADER_COUNT * FSEG_HEADER_SIZE)
-  end
-
-  # Return the byte offset of the start of the user records in a page, which
-  # immediately follows the supremum record.
-  def pos_user_records
-    pos_supremum + size_record_header + size_record_undefined + MUM_RECORD_SIZE
-  end
-
-  # Return the amount of free space in the page.
-  def free_space
-    unused_space = (PAGE_TRAILER_START - page_header[:heap_top])
-    unused_space + page_header[:garbage]
-  end
-
-  # Return the amount of used space in the page.
-  def used_space
-    PAGE_SIZE - free_space
-  end
-
-  # Return the amount of space occupied by records in the page.
-  def record_space
-    used_space - pos_user_records
-  end
-
-  # Return the actual bytes of the portion of the page which is used to
-  # store user records (eliminate the headers and trailer from the page).
-  def record_bytes
-    data(pos_user_records, page_header[:heap_top] - pos_user_records)
-  end
-
-  # Return the header from a record. (This is mostly unimplemented.)
-  def record_header(offset)
-    return nil unless type == :INDEX
-
-    c = cursor(offset).backward
-    case page_header[:format]
-    when :compact
-      header = {}
-      header[:next] = c.get_sint16
-      bits1 = c.get_uint16
-      header[:type] = bits1 & 0x07
-      header[:order] = (bits1 & 0xf8) >> 3
-      bits2 = c.get_uint8
-      header[:n_owned] = bits2 & 0x0f
-      header[:deleted] = (bits2 & 0xf0) >> 4
-      header
-    when :redundant
-      raise "Not implemented"
-    end
-  end
-
-  # Return a record. (This is mostly unimplemented.)
-  def record(offset)
-    return nil unless offset
-    return nil unless type == :INDEX
-    return nil if offset == pos_infimum
-    return nil if offset == pos_supremum
-
-    c = cursor(offset).forward
-    # There is a header preceding the row itself, so back up and read it.
-    header = record_header(offset)
-    {
-      :header => header,
-      :next => header[:next] == 0 ? nil : (offset + header[:next]),
-      # These system records may not be present depending on schema.
-      :rec1 => c.get_bytes(6),
-      :rec2 => c.get_bytes(6),
-      :rec3 => c.get_bytes(7),
-      # Read a few bytes just so it can be visually verified.
-      :data => c.get_bytes(8),
-    }
-  end
-
-  # Iterate through all records. (This is mostly unimplemented.)
-  def each_record
-    rec = infimum
-    while rec = record(rec[:next])
-      yield rec
-    end
-    nil
-  end
-
   # Dump the contents of a page for debugging purposes.
   def dump
     puts
-    puts "fil header:"
-    pp fil_header
+    puts "#{self}:"
 
     puts
-    puts "page header:"
-    pp page_header
-
-    puts
-    puts "free space: #{free_space}"
-    puts "used space: #{used_space}"
-    puts "record space: #{record_space}"
-
-    if type == :INDEX
-      puts
-      puts "system records:"
-      pp infimum
-      pp supremum
-      
-      puts
-      puts "records:"
-      each_record do |rec|
-        pp rec
-      end
-    end
+    puts "fil header:"
+    pp fil_header
   end
 end

data/lib/innodb/space.rb

@@ -1,21 +1,33 @@
 # An InnoDB tablespace file, which can be either a multi-table ibdataN file
 # or a single-table "innodb_file_per_table" .ibd file.
 class Innodb::Space
+  attr_accessor :record_formatter
+
+  # Currently only 16kB InnoDB pages are supported.
+  PAGE_SIZE = 16384
+
   # Open a tablespace file.
   def initialize(file)
     @file = File.open(file)
     @size = @file.stat.size
-    @pages = (@size / Innodb::Page::PAGE_SIZE)
+    @pages = (@size / PAGE_SIZE)
+    @record_formatter = nil
   end
 
   # Get an Innodb::Page object for a specific page by page number.
   def page(page_number)
-    offset = page_number.to_i * Innodb::Page::PAGE_SIZE
+    offset = page_number.to_i * PAGE_SIZE
     return nil unless offset < @size
-    return nil unless (offset + Innodb::Page::PAGE_SIZE) <= @size
+    return nil unless (offset + PAGE_SIZE) <= @size
     @file.seek(offset)
-    page_data = @file.read(Innodb::Page::PAGE_SIZE)
-    Innodb::Page.new(page_data)
+    page_data = @file.read(PAGE_SIZE)
+    this_page = Innodb::Page.parse(page_data)
+
+    if this_page.type == :INDEX
+      this_page.record_formatter = @record_formatter
+    end
+
+    this_page
   end
 
   # Iterate through all pages in a tablespace, returning the page number

data/lib/innodb/version.rb

@@ -1,3 +1,3 @@
 module Innodb
-  VERSION = "0.4"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: innodb_ruby
 version: !ruby/object:Gem::Version 
-  hash: 3
+  hash: 11
   prerelease: 
   segments: 
   - 0
-  - 4
-  version: "0.4"
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Jeremy Cole
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-11-20 00:00:00 Z
+date: 2012-11-29 00:00:00 Z
 dependencies: []
 
 description: Library for parsing InnoDB data files in Ruby