innodb_ruby 0.5.1 → 0.6.0

data/bin/innodb_space

@@ -0,0 +1,306 @@
+#!/usr/bin/env ruby
+
+require "getoptlong"
+require "ostruct"
+require "pp"
+require "innodb"
+
+def space_summary(space)
+  puts "%-12s%-20s%-12s%-12s%-20s" % [
+    "page",
+    "type",
+    "prev",
+    "next",
+    "lsn",
+  ]
+
+  space.each_page do |page_number, page|
+    puts "%-12i%-20s%-12i%-12i%-20i" % [
+      page_number,
+      page.type,
+      page.prev,
+      page.next,
+      page.lsn,
+    ]
+  end
+end
+
+def space_index_pages_summary(space)
+  puts "%-12s%-8s%-8s%-8s%-8s%-8s" % [
+    "page",
+    "index",
+    "level",
+    "data",
+    "free",
+    "records",
+  ]
+
+  space.each_page do |page_number, page|
+    case page.type
+    when :INDEX
+      puts "%-12i%-8i%-8i%-8i%-8i%-8i" % [
+        page_number,
+        page.ph[:index_id],
+        page.ph[:level],
+        page.record_space,
+        page.free_space,
+        page.ph[:n_recs],
+      ]
+    when :ALLOCATED
+      puts "%-12i%-8i%-8i%-8i%-8i%-8i" % [ page_number, 0, 0, 0, page.size, 0 ]
+    end
+  end
+end
+
+def index_recurse(index)
+  index.recurse(
+    lambda do |page, depth|
+      puts "%s%s %i: %i records, %i bytes" % [
+        "  " * depth,
+        page.level == 0 ? "LEAF" : "NODE",
+        page.offset,
+        page.page_header[:n_recs],
+        page.record_space,
+      ]
+      if page.level == 0
+        page.each_record do |record|
+          puts "%sRECORD: (%s) -> (%s)" % [
+            "  " * (depth+1),
+            record[:key].join(", "),
+            record[:row].join(", "),
+          ]
+        end
+      end
+    end,
+    lambda do |parent_page, child_page, child_min_key, depth|
+      puts "%sLINK %i -> %i: %s records >= (%s)" % [
+        "  " * depth,
+        parent_page.offset,
+        child_page.offset,
+        child_page.page_header[:n_recs],
+        child_min_key.join(", "),
+      ]
+    end
+  )
+end
+
+def index_digraph(index)
+  puts "digraph btree {"
+  puts "  rankdir = LR;"
+  puts "  ranksep = 2.0;"
+  index.recurse(
+    lambda do |page, depth|
+      label = "<page>Page %i|(%i records)" % [
+        page.offset,
+        page.page_header[:n_recs],
+      ]
+      page.each_child_page do |child_page_number, child_key|
+        label += "|<dir_%i>(%s)" % [
+          child_page_number,
+          child_key.join(", "),
+        ]
+      end
+      puts "  %spage_%i [ shape = \"record\"; label = \"%s\"; ];" % [
+        "  " * depth,
+        page.offset,
+        label,
+      ]
+    end,
+    lambda do |parent_page, child_page, child_key, depth|
+      puts "  %spage_%i:dir_%i -> page_%i:page:nw;" % [
+        "  " * depth,
+        parent_page.offset,
+        child_page.offset,
+        child_page.offset,
+      ]
+    end
+  )
+  puts "}"
+end
+
+def index_level_summary(index, levels)
+  puts "%-8s%-8s%-8s%-8s%-8s%-8s%-8s" % [
+    "page",
+    "index",
+    "level",
+    "data",
+    "free",
+    "records",
+    "min_key",
+  ]
+
+  levels.each do |level|
+    index.each_page_at_level(level) do |page|
+      puts "%-8i%-8i%-8i%-8i%-8i%-8i%s" % [
+        page.offset,
+        page.ph[:index_id],
+        page.ph[:level],
+        page.record_space,
+        page.free_space,
+        page.ph[:n_recs],
+        page.first_record[:key].join("|"),
+      ]
+    end
+  end
+end
+
+def usage(exit_code, message = nil)
+  print "Error: #{message}\n" unless message.nil?
+
+  print <<'END_OF_USAGE'
+
+Usage: innodb_space -f <file> [-p <page>] [-l <level>] <mode> [<mode>, ...]
+
+  --help, -?
+    Print this usage text.
+
+  --file, -f <file>
+    Load the tablespace file <file>.
+
+  --page, -p <page>
+    Operate on the page <page>; may be specified more than once.
+
+  --level, -l <level>
+    Operate on the level <level>; may be specified more than once.
+
+  --require, -r <file>
+    Use Ruby's "require" to load the file <file>. This is useful for loading
+    classes with record describers.
+
+  --describer, -d <describer>
+    Use the named record describer to parse records in index pages.
+
+The following modes are supported:
+
+  page-dump
+    Dump the contents of the page, using the Ruby pp ("pretty-print") module.
+
+  space-summary
+    Summarize all pages within a tablespace.
+
+  space-index-pages-summary
+    Summarize all "INDEX" pages within a tablespace. This is useful to analyze
+    page fill rates and record counts per page. In addition to "INDEX" pages,
+    "ALLOCATED" pages are also printed and assumed to be completely empty.
+
+  index-recurse
+    Recurse an index, starting at the root (which must be provided in the first
+    --page/-p argument), printing the node pages, node pointers (links), leaf
+    pages. A record describer must be provided with the --describer/-d argument
+    to recurse indexes (in order to parse node pages).
+
+  index-digraph
+    Recurse an index as index-recurse does, but print a dot-compatible digraph
+    instead of a human-readable summary.
+
+  index-level-summary
+    Print a summary of all pages at a given level (provided with the --level/-l
+    argument) in an index.
+
+END_OF_USAGE
+
+  exit exit_code
+end
+
+@options = OpenStruct.new
+@options.file   = nil
+@options.pages  = []
+@options.levels = []
+@options.describer = nil
+
+getopt_options = [
+  [ "--help",             "-?",     GetoptLong::NO_ARGUMENT ],
+  [ "--file",             "-f",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--page",             "-p",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--level",            "-l",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--require",          "-r",     GetoptLong::REQUIRED_ARGUMENT ],
+  [ "--describer",        "-d",     GetoptLong::REQUIRED_ARGUMENT ],
+]
+
+getopt = GetoptLong.new(*getopt_options)
+
+getopt.each do |opt, arg|
+  case opt
+  when "--help"
+    usage 0
+  when "--mode"
+    @options.mode = arg
+  when "--file"
+    @options.file = arg
+  when "--page"
+    @options.pages << arg.to_i
+  when "--level"
+    @options.levels << arg.to_i
+  when "--require"
+    require arg
+  when "--describer"
+    @options.describer = arg
+  end
+end
+
+unless @options.file
+  usage 1, "File must be provided with -f argument"
+end
+
+space = Innodb::Space.new(@options.file)
+
+if @options.describer
+  space.record_describer = Innodb::RecordDescriber.const_get(@options.describer)
+end
+
+if ARGV.empty?
+  usage 1, "At least one mode should be provided"
+end
+
+ARGV.each do |mode|
+  case mode
+  when "page-dump"
+    if @options.pages.empty?
+      usage 1, "Page numbers to dump must be provided"
+    end
+
+    @options.pages.each do |page|
+      space.page(page).dump
+    end
+  when "space-summary"
+    space_summary(space)
+  when "space-index-pages-summary"
+    space_index_pages_summary(space)
+  when "index-recurse"
+    unless space.record_describer
+      usage 1, "Record describer necessary for index recursion" 
+    end
+    
+    if @options.pages.empty?
+      usage 1, "Page number of index root must be provided"
+    end
+
+    @options.pages.each do |page|
+      index_recurse(space.index(page))
+    end
+  when "index-digraph"
+    unless space.record_describer
+      usage 1, "Record describer necessary for index recursion" 
+    end
+    
+    if @options.pages.empty?
+      usage 1, "Page number of index root must be provided"
+    end
+
+    @options.pages.each do |page|
+      index_digraph(space.index(page))
+    end
+  when "index-level-summary"
+    unless space.record_describer
+      usage 1, "Record describer necessary for index recursion" 
+    end
+    
+    if @options.pages.empty?
+      usage 1, "Page number of index root must be provided"
+    end
+
+    index_level_summary(space.index(@options.pages.first), @options.levels)
+  else
+    usage 1, "Unknown mode: #{mode}"
+  end
+end

data/lib/innodb.rb

@@ -6,6 +6,7 @@ require "innodb/page"
 require "innodb/page/fsp_hdr_xdes"
 require "innodb/page/inode"
 require "innodb/page/index"
+require "innodb/record_describer"
 require "innodb/space"
 require "innodb/index"
 require "innodb/log_block"

data/lib/innodb/index.rb

@@ -32,7 +32,7 @@ class Innodb::Index
 
     parent_page.each_child_page do |child_page_number, child_min_key|
       child_page = @space.page(child_page_number)
-      child_page.record_formatter = @space.record_formatter
+      child_page.record_describer = @space.record_describer
       if child_page.type == :INDEX
         if link_proc
           link_proc.call(parent_page, child_page, child_min_key, depth+1)
@@ -76,7 +76,7 @@ class Innodb::Index
 
   # Iterate through all records on all leaf pages in ascending order.
   def each_record
-    each_leaf_page do |page|
+    each_page_at_level(0) do |page|
       page.each_record do |record|
         yield record
       end

data/lib/innodb/log.rb

@@ -16,10 +16,14 @@ class Innodb::Log
 
   # Open a log file.
   def initialize(file)
-    @file = File.open(file)
-    @size = @file.stat.size
-    @blocks = ((@size - DATA_START) / Innodb::LogBlock::BLOCK_SIZE)
+    @name = file
+    File.open(@name) do |log|
+      @size = log.stat.size
+      @blocks = ((@size - DATA_START) / Innodb::LogBlock::BLOCK_SIZE)
+    end
   end
+  
+  attr_reader :blocks
 
   # Return a log block with a given block number as an InnoDB::LogBlock object.
   # Blocks are numbered after the log file header, starting from 0.
@@ -27,9 +31,11 @@ class Innodb::Log
     offset = DATA_START + (block_number.to_i * Innodb::LogBlock::BLOCK_SIZE)
     return nil unless offset < @size
     return nil unless (offset + Innodb::LogBlock::BLOCK_SIZE) <= @size
-    @file.seek(offset)
-    block_data = @file.read(Innodb::LogBlock::BLOCK_SIZE)
-    Innodb::LogBlock.new(block_data)
+    File.open(@name) do |log|
+      log.seek(offset)
+      block_data = log.read(Innodb::LogBlock::BLOCK_SIZE)
+      Innodb::LogBlock.new(block_data)
+    end
   end
 
   # Iterate through all log blocks, returning the block number and an

data/lib/innodb/log_block.rb

@@ -57,7 +57,7 @@ class Innodb::LogBlock
     @header ||= begin
       c = cursor(HEADER_START)
       {
-        :block            => c.get_uint32,
+        :block            => c.get_uint32 & (2**31-1),
         :data_length      => c.get_uint16,
         :first_rec_group  => c.get_uint16,
         :checkpoint_no    => c.get_uint32,
@@ -143,7 +143,7 @@ class Innodb::LogBlock
   end
 
   SINGLE_RECORD_MASK = 0x80
-  RECORD_TYPE_MASK = 0x7f
+  RECORD_TYPE_MASK   = 0x7f
 
   # Return the log record. (This is mostly unimplemented.)
   def record
@@ -153,7 +153,7 @@ class Innodb::LogBlock
         type_and_flag = c.get_uint8
         type = type_and_flag & RECORD_TYPE_MASK
         type = RECORD_TYPES[type] || type
-        single_record = (type_and_flag & SINGLE_RECORD_MASK) == SINGLE_RECORD_MASK
+        single_record = (type_and_flag & SINGLE_RECORD_MASK) > 0
         {
           :type           => type,
           :single_record  => single_record,

data/lib/innodb/page.rb

@@ -126,13 +126,18 @@ class Innodb::Page
     fil_header[:next]
   end
 
+  # A helper function to return the LSN, for easier access.
+  def lsn
+    fil_header[:lsn]
+  end
+
   # Dump the contents of a page for debugging purposes.
   def dump
-    puts
     puts "#{self}:"
-
     puts
+
     puts "fil header:"
     pp fil_header
+    puts
   end
 end

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -62,15 +62,15 @@ class Innodb::Page::FspHdrXdes < Innodb::Page
   def dump
     super
 
-    puts
     puts "fsp header:"
     pp fsp_header
-    
     puts
+
     puts "xdes entries:"
     each_xdes do |xdes|
       pp xdes
     end
+    puts
   end
 end
 

data/lib/innodb/page/index.rb

@@ -1,5 +1,5 @@
 class Innodb::Page::Index < Innodb::Page
-  attr_accessor :record_formatter
+  attr_accessor :record_describer
 
   # Return the byte offset of the start of the "index" page header, which
   # immediately follows the "fil" header.
@@ -229,8 +229,8 @@ class Innodb::Page::Index < Innodb::Page
 
   # Return (and cache) the record format provided by an external class.
   def record_format
-    if record_formatter
-      @record_format ||= record_formatter.format(self)
+    if record_describer
+      @record_format ||= record_describer.cursor_sendable_description(self)
     end
   end
 
@@ -318,11 +318,10 @@ class Innodb::Page::Index < Innodb::Page
   def dump
     super
 
-    puts
     puts "page header:"
     pp page_header
-
     puts
+
     puts "sizes:"
     puts "  %-15s%5i" % [ "header",     header_space ]
     puts "  %-15s%5i" % [ "trailer",    trailer_space ]
@@ -334,17 +333,18 @@ class Innodb::Page::Index < Innodb::Page
       "per record",
       (page_header[:n_recs] > 0) ? (record_space / page_header[:n_recs]) : 0
     ]
-
     puts
+
     puts "system records:"
     pp infimum
     pp supremum
-    
     puts
+
     puts "records:"
     each_record do |rec|
       pp rec
     end
+    puts
   end
 end
 

data/lib/innodb/page/inode.rb

@@ -35,9 +35,9 @@ class Innodb::Page::Inode < Innodb::Page
   def dump
     super
 
-    puts
     puts "inode header:"
     pp inode_header
+    puts
   end
 end
 

data/lib/innodb/record_describer.rb

@@ -0,0 +1 @@
+class Innodb::RecordDescriber; end

data/lib/innodb/space.rb

@@ -1,7 +1,7 @@
 # An InnoDB tablespace file, which can be either a multi-table ibdataN file
 # or a single-table "innodb_file_per_table" .ibd file.
 class Innodb::Space
-  attr_accessor :record_formatter
+  attr_accessor :record_describer
 
   # Currently only 16kB InnoDB pages are supported.
   PAGE_SIZE = 16384
@@ -11,7 +11,7 @@ class Innodb::Space
     @file = File.open(file)
     @size = @file.stat.size
     @pages = (@size / PAGE_SIZE)
-    @record_formatter = nil
+    @record_describer = nil
   end
 
   # Get an Innodb::Page object for a specific page by page number.
@@ -24,12 +24,17 @@ class Innodb::Space
     this_page = Innodb::Page.parse(page_data)
 
     if this_page.type == :INDEX
-      this_page.record_formatter = @record_formatter
+      this_page.record_describer = @record_describer
     end
 
     this_page
   end
 
+  # Get an Innodb::Index object for a specific index by root page number.
+  def index(root_page_number)
+    Innodb::Index.new(self, root_page_number)
+  end
+
   # Iterate through all pages in a tablespace, returning the page number
   # and an Innodb::Page object for each one.
   def each_page

data/lib/innodb/version.rb

@@ -1,3 +1,3 @@
 module Innodb
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: innodb_ruby
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 7
   prerelease: 
   segments: 
   - 0
-  - 5
-  - 1
-  version: 0.5.1
+  - 6
+  - 0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Jeremy Cole
@@ -15,14 +15,14 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-11-29 00:00:00 Z
+date: 2012-12-01 00:00:00 Z
 dependencies: []
 
 description: Library for parsing InnoDB data files in Ruby
 email: jeremy@jcole.us
 executables: 
-- innodb_dump_log
-- innodb_dump_space
+- innodb_log
+- innodb_space
 extensions: []
 
 extra_rdoc_files: []
@@ -38,10 +38,11 @@ files:
 - lib/innodb/page/fsp_hdr_xdes.rb
 - lib/innodb/page/index.rb
 - lib/innodb/page/inode.rb
+- lib/innodb/record_describer.rb
 - lib/innodb/space.rb
 - lib/innodb/version.rb
-- bin/innodb_dump_log
-- bin/innodb_dump_space
+- bin/innodb_log
+- bin/innodb_space
 homepage: http://jcole.us/
 licenses: []
 

data/bin/innodb_dump_space

@@ -1,189 +0,0 @@
-#!/usr/bin/env ruby
-
-require "pp"
-require "innodb"
-
-class EdgesRecordFormatter
-  def format(page)
-    bytes_per_record = (page.record_space / page.page_header[:n_recs])
-    case
-    when [48, 26].include?(bytes_per_record) # Clustered Key
-      {
-        # PRIMARY KEY (source_id, state, position)
-        :type => :clustered,
-        :key => [
-          [:get_uint64],    # source_id
-          [:get_i_sint8],   # state
-          [:get_i_sint64],  # position
-        ],
-        :row => [
-          [:get_uint32],    # updated_at
-          [:get_uint64],    # destination_id
-          [:get_uint8],     # count
-        ],
-      }
-    when [30, 34].include?(bytes_per_record) # Secondary Key
-      {
-        # INDEX (source_id, destination_id)
-        :type => :secondary,
-        :key => [
-          [:get_uint64],    # source_id
-          [:get_uint64],    # destination_id
-        ],
-        # PKV ([source_id], state, position)
-        :row => [
-          [:get_i_sint8],   # state
-          [:get_i_sint64],  # position
-        ],
-      }
-    end
-  end
-end
-
-class PageSplitTestFormatter
-  def format(page)
-      {
-      # PRIMARY KEY (id)
-      :type => :clustered,
-      :key => [
-        [:get_uint64],  # id
-      ],
-      :row => [],
-    }
-  end
-end
-
-def digraph_index(index)
-  puts "digraph btree {"
-  puts "  rankdir = LR;"
-  puts "  ranksep = 2.0;"
-  index.recurse(
-    lambda do |page, depth|
-      if true #page.level != 0
-        label = "<page>Page %i|(%i records)" % [
-          page.offset,
-          page.page_header[:n_recs],
-        ]
-        page.each_child_page do |child_page_number, child_key|
-          label += "|<dir_%i>(%s)" % [
-            child_page_number,
-            child_key.join(", "),
-          ]
-        end
-        puts "  %spage_%i [ shape = \"record\"; label = \"%s\"; ];" % [
-          "  " * depth,
-          page.offset,
-          label,
-        ]
-      end
-    end,
-    lambda do |parent_page, child_page, child_key, depth|
-      if true #child_page.level > 0
-        puts "  %spage_%i:dir_%i -> page_%i:page:nw;" % [
-          "  " * depth,
-          parent_page.offset,
-          child_page.offset,
-          child_page.offset,
-        ]
-      end
-    end
-  )
-  puts "}"
-end
-
-def recurse_index(index)
-  index.recurse(
-    lambda do |page, depth|
-      puts "%s%s %i: %i records, %i bytes" % [
-        "  " * depth,
-        page.level == 0 ? "LEAF" : "NODE",
-        page.offset,
-        page.page_header[:n_recs],
-        page.record_space,
-      ]
-      if page.level == 0
-        page.each_record do |record|
-          puts "%sRECORD: (%s) -> (%s)" % [
-            "  " * (depth+1),
-            record[:key].join(", "),
-            record[:row].join(", "),
-          ]
-        end
-      end
-    end,
-    lambda do |parent_page, child_page, child_min_key, depth|
-      puts "%sLINK %i -> %i: %s records >= (%s)" % [
-        "  " * depth,
-        parent_page.offset,
-        child_page.offset,
-        child_page.page_header[:n_recs],
-        child_min_key.join(", "),
-      ]
-    end
-  )
-end
-
-def dump_space(space)
-  puts "%-8s%-8s%-8s%-8s%-8s%-8s" % [
-    "page",
-    "index",
-    "level",
-    "data",
-    "free",
-    "records",
-  ]
-
-  space.each_page do |page_number, page|
-    case page.type
-    when :INDEX
-      puts "%-8i%-8i%-8i%-8i%-8i%-8i" % [
-        page_number,
-        page.ph[:index_id],
-        page.ph[:level],
-        page.record_space,
-        page.free_space,
-        page.ph[:n_recs],
-      ]
-    when :ALLOCATED
-      puts "%-8i%-8i%-8i%-8i%-8i%-8i" % [ page_number, 0, 0, 0, Innodb::Space::PAGE_SIZE, 0 ]
-    end
-  end
-end
-
-def dump_index_level(index, level)
-  puts "%-8s%-8s%-8s%-8s%-8s%-8s%-8s" % [
-    "page",
-    "index",
-    "level",
-    "data",
-    "free",
-    "records",
-    "min_key",
-  ]
-
-  index.each_page_at_level(level) do |page|
-    puts "%-8i%-8i%-8i%-8i%-8i%-8i%s" % [
-      page.offset,
-      page.ph[:index_id],
-      page.ph[:level],
-      page.record_space,
-      page.free_space,
-      page.ph[:n_recs],
-      page.first_record[:key].join("|"),
-    ]
-  end
-end
-
-file, page_number = ARGV.shift(2)
-
-formatter = EdgesRecordFormatter.new
-#formatter = PageSplitTestFormatter.new
-space = Innodb::Space.new(file)
-space.record_formatter = formatter
-space.page(page_number).dump
-#space.each_page { |page_number, page| puts "%6i%20s" % [page_number, page.type] }
-#dump_space(space)
-#dump_index_level(Innodb::Index.new(space, page_number), 1)
-#recurse_index(Innodb::Index.new(space, page_number))
-#digraph_index(Innodb::Index.new(space, page_number))
-#Innodb::Index.new(space, page_number).each_record { |record| puts "(#{record[:key].join(", ")}) -> (#{record[:row].join(", ")})" }