innodb_ruby 0.12.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2dd793281e3b83403e55c8d4e9104898d5df6099af80dd4453795a524e3fb32
-  data.tar.gz: d6ac28ab11759a618f9a84af611a705644f5f47f9c1cd387360b628bac88f527
+  metadata.gz: 65c78efba30c942c5529781a0dc7b36816af82d8aa4077d29d535f0727485c02
+  data.tar.gz: 6eb04e7370deee2f220383b36dd9d1dd1d8c454024f7a20fa2a3a8ece386bb14
 SHA512:
-  metadata.gz: ffb5ba4b9805d4cb78aaecaec92adb4e6064d73a9349fbeb43fcc938b958adeb14f140747dd540f76d31bbf9a38fbde259bd99f3a9cc64f32a1456efc2006997
-  data.tar.gz: 9e98bd61fe4448fdbb15be66975581fe65d97c342685c21f2309ead8b0ed1fc217b08baf2855c336d8ea426e22b7c0556e0926064722e6cc23a95312a6a63ca4
+  metadata.gz: ac98dfc4efeb3e42941bba41b021037e480e53a25300f3d4d115eae44e2c6435592f00f2d65934a0e14166e93fc2dd9e376189bd0c8945a138b0e4ece3a16b59
+  data.tar.gz: 449fb6bd1c19690a52aaa065efa6f2465a5aae66be3e1a42b69c2826c8d489babd4979e5ef2c8e840b81629c965d0731da72bc358845306e76b15efa81c34b80

data/bin/innodb_log

@@ -2,7 +2,6 @@
 # frozen_string_literal: true
 
 require "getoptlong"
-require "ostruct"
 require "set"
 require "innodb"
 
@@ -60,7 +59,7 @@ def usage(exit_code, message = nil)
   print "Error: #{message}\n" unless message.nil?
 
   # rubocop:disable Layout/HeredocIndentation
-  print <<'END_OF_USAGE'
+  print <<END_OF_USAGE
 
 Usage: innodb_log [-d] [-l <lsn>] -f <log file> <mode>
 
@@ -96,7 +95,14 @@ END_OF_USAGE
   exit exit_code
 end
 
-@options = OpenStruct.new
+InnodbLogOptions = Struct.new(
+  :log_files,
+  :dump,
+  :lsn,
+  keyword_init: true
+)
+
+@options = InnodbLogOptions.new
 @options.log_files = []
 @options.dump = false
 @options.lsn = nil

data/bin/innodb_space

@@ -2,7 +2,6 @@
 # frozen_string_literal: true
 
 require "getoptlong"
-require "ostruct"
 require "histogram/array"
 require "innodb"
 
@@ -118,10 +117,10 @@ def print_lists(lists)
     puts "%-20s%-12i%-12i%-12i%-12i%-12i" % [
       name,
       list.base[:length],
-      list.base[:first] && list.base[:first][:page]   || 0,
-      list.base[:first] && list.base[:first][:offset] || 0,
-      list.base[:last]  && list.base[:last][:page]    || 0,
-      list.base[:last]  && list.base[:last][:offset]  || 0,
+      (list.base[:first] && list.base[:first][:page])   || 0,
+      (list.base[:first] && list.base[:first][:offset]) || 0,
+      (list.base[:last]  && list.base[:last][:page])    || 0,
+      (list.base[:last]  && list.base[:last][:offset])  || 0,
     ]
   end
 end
@@ -157,8 +156,8 @@ def print_index_page_summary(pages)
   ]
 
   pages.each do |page_number, page|
-    case page.type
-    when :INDEX
+    case
+    when page.is_a?(Innodb::Page::Index)
       puts "%-12i%-8i%-8i%-8i%-8i%-8i" % [
         page_number,
         page.page_header[:index_id],
@@ -167,7 +166,7 @@ def print_index_page_summary(pages)
         page.free_space,
         page.records,
       ]
-    when :ALLOCATED
+    when page.type == :ALLOCATED
       puts "%-12i%-8i%-8i%-8i%-8i%-8i" % [page_number, 0, 0, 0, page.size, 0]
     end
   end
@@ -430,12 +429,12 @@ def space_index_pages_free_plot(space, start_page)
   index_data = { 0 => { x: [], y: [] } }
 
   space.each_page(start_page) do |page_number, page|
-    case page.type
-    when :INDEX
+    case
+    when page.is_a?(Innodb::Page::Index)
       data = (index_data[page.page_header[:index_id]] ||= { x: [], y: [] })
       data[:x] << page_number
       data[:y] << page.free_space
-    when :ALLOCATED
+    when page.type == :ALLOCATED
       index_data[0][:x] << page_number
       index_data[0][:y] << page.size
     end
@@ -445,7 +444,7 @@ def space_index_pages_free_plot(space, start_page)
   image_file = "#{image_name}_free.png"
 
   # Aim for one horizontal pixel per extent, but min 1k and max 10k width.
-  image_width = [10_000, [1_000, space.pages / space.pages_per_extent].max].min
+  (space.pages / space.pages_per_extent).clamp(1_000, 10_000)
 
   Gnuplot.open do |gp|
     Gnuplot::Plot.new(gp) do |plot|
@@ -476,7 +475,7 @@ end
 
 # rubocop:disable Metrics/BlockNesting
 def space_extents_illustrate_page_status(space, entry, count_by_identifier, identifiers)
-  entry.each_page_status.each_with_object("".dup) do |(page_number, page_status), bitmap|
+  entry.each_page_status.with_object("".dup) do |(page_number, page_status), bitmap|
     if page_number >= space.pages
       bitmap << " "
       next
@@ -577,8 +576,8 @@ def space_lsn_age_illustrate(space)
     next if page.lsn.zero?
 
     page_lsn[page_number] = page.lsn
-    lsn_min = page.lsn < lsn_min ? page.lsn : lsn_min
-    lsn_max = page.lsn > lsn_max ? page.lsn : lsn_max
+    lsn_min = [page.lsn, lsn_min].min
+    lsn_max = [page.lsn, lsn_max].max
   end
   lsn_delta = lsn_max - lsn_min
 
@@ -606,7 +605,7 @@ def space_lsn_age_illustrate(space)
 
   puts "%12s ╰%-#{width}s╯" % ["", "─" * width]
 
-  _, lsn_freq = page_lsn.reject(&:nil?).histogram(colors.size, min: lsn_min, max: lsn_max)
+  _, lsn_freq = page_lsn.compact.histogram(colors.size, min: lsn_min, max: lsn_max)
   lsn_freq_delta = lsn_freq.max - lsn_freq.min
 
   lsn_age_histogram = "".dup
@@ -690,10 +689,9 @@ def page_account(innodb_system, space, page_number)
 
   page = space.page(page_number)
   page_type = Innodb::Page::PAGE_TYPE[page.type]
-  puts "  Page type is %s (%s, %s)." % [
+  puts "  Page type is %s (%s)." % [
     page.type,
     page_type[:description],
-    page_type[:usage],
   ]
 
   xdes = space.xdes_for_page(page_number)
@@ -922,7 +920,7 @@ def page_validate(_innodb_system, space, page_number)
     end
   end
 
-  page_is_valid = false if page.type == :INDEX && !page_validate_index(page)
+  page_is_valid = false if page.is_a?(Innodb::Page::Index) && !page_validate_index(page)
 
   puts "Page %d appears to be %s!" % [
     page_number,
@@ -931,7 +929,7 @@ def page_validate(_innodb_system, space, page_number)
 end
 
 def page_directory_summary(_space, page)
-  usage(1, "Page must be an index page") if page.type != :INDEX
+  usage(1, "Page must be an index page") unless page.is_a?(Innodb::Page::Index)
 
   puts "%-8s%-8s%-14s%-8s%s" % %w[
     slot
@@ -966,7 +964,7 @@ end
 
 def page_illustrate(page)
   width = 64
-  unknown_page_content = page.type == :INDEX && page.record_describer.nil?
+  unknown_page_content = page.is_a?(Innodb::Page::Index) && page.record_describer.nil?
   blocks = Array.new(page.size, unknown_page_content ? "▞" : " ")
   identifiers = {}
   identifier_sort = 0
@@ -1251,7 +1249,7 @@ def usage(exit_code, message = nil)
   end
 
   # rubocop:disable Layout/HeredocIndentation
-  print <<'END_OF_USAGE'
+  print <<END_OF_USAGE
 
 Usage: innodb_space <options> <mode>
 
@@ -1484,7 +1482,26 @@ end
   Signal.trap(name) { exit } if Signal.list.include?(name)
 end
 
-@options = OpenStruct.new
+InnodbSpaceOptions = Struct.new(
+  :trace,
+  :system_space_file,
+  :system_space_tables,
+  :data_directory,
+  :space_file,
+  :table_name,
+  :index_name,
+  :page,
+  :record,
+  :level,
+  :list,
+  :fseg_id,
+  :describer,
+  :illustration_line_width,
+  :illustration_block_size,
+  keyword_init: true
+)
+
+@options = InnodbSpaceOptions.new
 @options.trace                    = 0
 @options.system_space_file        = nil
 @options.system_space_tables      = false
@@ -1625,7 +1642,7 @@ if innodb_system && @options.table_name && @options.index_name
   page = @options.page ? space.page(@options.page) : index.root
 elsif @options.page
   page = space.page(@options.page)
-  index = space.index(@options.page) if page&.type == :INDEX && page&.root?
+  index = space.index(@options.page) if page.is_a?(Innodb::Page::Index) && page&.root?
 end
 
 # The non-option argument on the command line is the mode (usually the last,
@@ -1678,7 +1695,7 @@ if /^record-/.match(mode) && !page
   )
 end
 
-if /^record-/.match(mode) && page.type != :INDEX
+if /^record-/.match(mode) && !page.is_a?(Innodb::Page::Index)
   usage(1, "Mode #{mode} may be used only with index pages")
 end
 

data/lib/innodb/data_dictionary.rb

@@ -235,7 +235,14 @@ module Innodb
       system_space.data_dictionary_page.data_dictionary_header[:indexes]
     end
 
+    # Check if the data dictionary indexes are all available.
+    def found?
+      data_dictionary_indexes.values.map(&:values).flatten.none?(&:zero?)
+    end
+
     def data_dictionary_index_ids
+      raise "Data Dictionary not found; is the MySQL version supported?" unless found?
+
       return @data_dictionary_index_ids if @data_dictionary_index_ids
 
       # TODO: This could probably be done a lot more Ruby-like.
@@ -260,7 +267,7 @@ module Innodb
     end
 
     def data_dictionary_index?(table_name, index_name)
-      return unless data_dictionary_table?(table_name)
+      return false unless data_dictionary_table?(table_name)
 
       DATA_DICTIONARY_RECORD_DESCRIBERS[table_name.to_sym].include?(index_name.to_sym)
     end
@@ -275,6 +282,8 @@ module Innodb
     # internal data dictionary index with an appropriate
     # record describer so that records can be recursed.
     def data_dictionary_index(table_name, index_name)
+      raise "Data Dictionary not found; is the MySQL version supported?" unless found?
+
       table_entry = data_dictionary_indexes[table_name]
       raise "Unknown data dictionary table #{table_name}" unless table_entry
 
@@ -307,7 +316,7 @@ module Innodb
       return enum_for(:each_data_dictionary_index) unless block_given?
 
       data_dictionary_indexes.each do |table_name, indexes|
-        indexes.each do |index_name, _root_page_number|
+        indexes.each_key do |index_name|
           yield table_name, index_name,
             data_dictionary_index(table_name, index_name)
         end

data/lib/innodb/data_type.rb

@@ -112,8 +112,8 @@ module Innodb
         @uncomp_fractional = scale / MAX_DIGITS_PER_INTEGER
         @comp_integral = integral - (@uncomp_integral * MAX_DIGITS_PER_INTEGER)
         @comp_fractional = scale - (@uncomp_fractional * MAX_DIGITS_PER_INTEGER)
-        @width = @uncomp_integral * 4 + BYTES_PER_DIGIT[@comp_integral] +
-                 @comp_fractional * 4 + BYTES_PER_DIGIT[@comp_fractional]
+        @width = (@uncomp_integral * 4) + BYTES_PER_DIGIT[@comp_integral] +
+                 (@comp_fractional * 4) + BYTES_PER_DIGIT[@comp_fractional]
         @name = Innodb::DataType.make_name(base_type, modifiers, properties)
       end
 

data/lib/innodb/field.rb

@@ -121,7 +121,7 @@ module Innodb
       base_type = matches[1].upcase.to_sym
       return [base_type, []] unless matches[3]
 
-      [base_type, matches[3].sub(/ /, "").split(/,/).map(&:to_i)]
+      [base_type, matches[3].sub(/ /, "").split(",").map(&:to_i)]
     end
   end
 end

data/lib/innodb/index.rb

@@ -21,7 +21,9 @@ module Innodb
       raise "Page #{root_page_number} couldn't be read" unless @root
 
       # The root page should be an index page.
-      raise "Page #{root_page_number} is a #{@root.type} page, not an INDEX page" unless @root.type == :INDEX
+      unless @root.is_a?(Innodb::Page::Index)
+        raise "Page #{root_page_number} is a #{@root.type} page, not an INDEX page"
+      end
 
       # The root page should be the only page at its level.
       raise "Page #{root_page_number} does not appear to be an index root" if @root.prev || @root.next
@@ -53,12 +55,12 @@ module Innodb
 
     # Internal method used by recurse.
     def _recurse(parent_page, page_proc, link_proc, depth = 0)
-      page_proc.call(parent_page, depth) if page_proc && parent_page.type == :INDEX
+      page_proc.call(parent_page, depth) if page_proc && parent_page.is_a?(Innodb::Page::Index)
 
       parent_page.each_child_page do |child_page_number, child_min_key|
         child_page = page(child_page_number)
         child_page.record_describer = record_describer
-        next unless child_page.type == :INDEX
+        next unless child_page.is_a?(Innodb::Page::Index)
 
         link_proc&.call(parent_page, child_page, child_min_key, depth + 1)
         _recurse(child_page, page_proc, link_proc, depth + 1)
@@ -143,7 +145,7 @@ module Innodb
     def each_page_from(page)
       return enum_for(:each_page_from, page) unless block_given?
 
-      while page && page.type == :INDEX
+      while page.is_a?(Innodb::Page::Index)
         yield page
         break unless page.next
 

data/lib/innodb/inode.rb

@@ -99,7 +99,7 @@ module Innodb
 
     # Helper method to return an array of only non-nil fragment pages.
     def frag_array_pages
-      frag_array.reject(&:nil?)
+      frag_array.compact
     end
 
     # Helper method to count non-nil fragment pages.

data/lib/innodb/lsn.rb

@@ -40,7 +40,7 @@ module Innodb
       fragment = (@lsn_no % LOG_BLOCK_SIZE) - LOG_BLOCK_HEADER_SIZE
       raise "Invalid fragment #{fragment} for LSN #{@lsn_no}" unless fragment.between?(0, LOG_BLOCK_DATA_SIZE - 1)
 
-      length + (fragment + length) / LOG_BLOCK_DATA_SIZE * LOG_BLOCK_FRAME_SIZE
+      length + ((fragment + length) / LOG_BLOCK_DATA_SIZE * LOG_BLOCK_FRAME_SIZE)
     end
 
     # Whether LSN might point to log record data.
@@ -82,12 +82,12 @@ module Innodb
       end
 
       # Transpose group size offset to a group capacity offset.
-      group_offset = offset - (LOG_HEADER_SIZE * (1 + offset / log_size))
+      group_offset = offset - (LOG_HEADER_SIZE * (1 + (offset / log_size)))
 
       offset = (lsn_offset + group_offset) % group_capacity
 
       # Transpose group capacity offset to a group size offset.
-      offset + LOG_HEADER_SIZE * (1 + offset / (log_size - LOG_HEADER_SIZE))
+      offset + (LOG_HEADER_SIZE * (1 + (offset / (log_size - LOG_HEADER_SIZE))))
     end
 
     # Whether offset points to the data area of an existing log block.

data/lib/innodb/page/blob.rb

@@ -32,22 +32,16 @@ module Innodb
         end
       end
 
-      def dump_hex(string)
-        slice_size = 16
-        string.chars.each_slice(slice_size).each_with_index do |slice_bytes, slice_count|
-          puts "%08i  %-23s  %-23s  |%-16s|" % [
-            (slice_count * slice_size),
-            slice_bytes[0..8].map { |n| "%02x" % n.ord }.join(" "),
-            slice_bytes[8..16].map { |n| "%02x" % n.ord }.join(" "),
-            slice_bytes.join,
-          ]
-        end
+      def next_blob_page
+        return unless blob_header[:next]
+
+        space.page(blob_header[:next])
       end
 
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_blob_header,
@@ -75,7 +69,7 @@ module Innodb
         puts
 
         puts "blob data:"
-        dump_hex(blob_data)
+        HexFormat.puts(blob_data)
         puts
 
         puts

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -47,6 +47,22 @@ module Innodb
         keyword_init: true
       )
 
+      EncryptionHeader = Struct.new(
+        :magic,
+        :master_key_id,
+        :key,
+        :iv,
+        :server_uuid,
+        :checksum,
+        keyword_init: true
+      )
+
+      SdiHeader = Struct.new(
+        :version,
+        :root_page_number,
+        keyword_init: true
+      )
+
       # A value added to the adjusted exponent stored in the page size field of
       # the flags in the FSP header.
       FLAGS_PAGE_SIZE_SHIFT = 9
@@ -118,6 +134,22 @@ module Innodb
         entries_in_xdes_array * size_xdes_entry
       end
 
+      def pos_encryption_header
+        pos_xdes_array + size_xdes_array
+      end
+
+      def size_encryption_header
+        3 + 4 + (32 * 2) + 36 + 4 + 4
+      end
+
+      def pos_sdi_header
+        pos_encryption_header + size_encryption_header
+      end
+
+      def size_sdi_header
+        8
+      end
+
       # Read the FSP (filespace) header, which contains a few counters and flags,
       # as well as list base nodes for each list maintained in the filespace.
       def fsp_header
@@ -162,10 +194,32 @@ module Innodb
         end
       end
 
+      def encryption_header
+        @encryption_header ||= cursor(pos_encryption_header).name("encryption_header") do |c|
+          EncryptionHeader.new(
+            magic: c.name("magic") { c.read_bytes(3) },
+            master_key_id: c.name("master_key_id") { c.read_uint32 },
+            key: c.name("key") { c.read_bytes(32) },
+            iv: c.name("iv") { c.read_bytes(32) },
+            server_uuid: c.name("server_uuid") { c.read_string(36) },
+            checksum: c.name("checksum") { c.read_uint32 }
+          )
+        end
+      end
+
+      def sdi_header
+        @sdi_header ||= cursor(pos_sdi_header).name("sdi_header") do |c|
+          SdiHeader.new(
+            version: c.name("version") { c.read_uint32 },
+            root_page_number: c.name("root_page_number") { c.read_uint32 }
+          )
+        end
+      end
+
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_fsp_header,
@@ -179,11 +233,25 @@ module Innodb
           yield Region.new(
             offset: xdes.offset,
             length: size_xdes_entry,
-            name: "xdes_#{state}".to_sym,
+            name: :"xdes_#{state}",
             info: "Extent Descriptor (#{state})"
           )
         end
 
+        yield Region.new(
+          offset: pos_encryption_header,
+          length: size_encryption_header,
+          name: :encryption_header,
+          info: "Encryption Header"
+        )
+
+        yield Region.new(
+          offset: pos_sdi_header,
+          length: size_sdi_header,
+          name: :sdi_header,
+          info: "SDI Header"
+        )
+
         nil
       end
 
@@ -200,6 +268,14 @@ module Innodb
           pp xdes
         end
         puts
+
+        puts "encryption header:"
+        pp encryption_header
+        puts
+
+        puts "serialized dictionary information header:"
+        pp sdi_header
+        puts
       end
     end
   end

data/lib/innodb/page/ibuf_bitmap.rb

@@ -22,7 +22,7 @@ module Innodb
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_ibuf_bitmap,

data/lib/innodb/page/index.rb

@@ -130,7 +130,7 @@ module Innodb
       # Maximum number of fields.
       RECORD_MAX_N_SYSTEM_FIELDS  = 3
       RECORD_MAX_N_FIELDS         = 1024 - 1
-      RECORD_MAX_N_USER_FIELDS    = RECORD_MAX_N_FIELDS - RECORD_MAX_N_SYSTEM_FIELDS * 2
+      RECORD_MAX_N_USER_FIELDS    = RECORD_MAX_N_FIELDS - (RECORD_MAX_N_SYSTEM_FIELDS * 2)
 
       # Page direction values possible in the page_header's :direction field.
       PAGE_DIRECTION = {
@@ -314,8 +314,8 @@ module Innodb
             index_id: c.name("index_id") { c.read_uint64 }
           )
 
-          index.n_heap = index.n_heap_format & (2**15 - 1)
-          index.format = (index.n_heap_format & 1 << 15).zero? ? :redundant : :compact
+          index.n_heap = index.n_heap_format & ((2**15) - 1)
+          index.format = (index.n_heap_format & (1 << 15)).zero? ? :redundant : :compact
 
           index
         end
@@ -546,7 +546,7 @@ module Innodb
       end
 
       def make_record_describer
-        if space&.innodb_system && index_id
+        if space&.innodb_system&.data_dictionary&.found? && index_id && !ibuf_index?
           @record_describer = space.innodb_system.data_dictionary.record_describer_by_index_id(index_id)
         elsif space
           @record_describer = space.record_describer
@@ -998,7 +998,7 @@ module Innodb
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_index_header,
@@ -1096,17 +1096,23 @@ module Innodb
         pp supremum.record
         puts
 
-        puts "garbage records:"
-        each_garbage_record do |rec|
-          pp rec.record
+        if ibuf_index?
+          puts "(records not dumped due to this being an insert buffer index)"
+        elsif !record_describer
+          puts "(records not dumped due to missing record describer or data dictionary)"
+        else
+          puts "garbage records:"
+          each_garbage_record do |rec|
+            pp rec.record
+            puts
+          end
           puts
-        end
-        puts
 
-        puts "records:"
-        each_record do |rec|
-          pp rec.record
-          puts
+          puts "records:"
+          each_record do |rec|
+            pp rec.record
+            puts
+          end
         end
         puts
       end

data/lib/innodb/page/index_compressed.rb

@@ -34,7 +34,7 @@ module Innodb
       def free_space
         free_space_start =
           size - size_fil_trailer - directory_space - (uncompressed_columns_size * (page_header.n_heap - 2))
-        puts "Free space start == %04x" % [offset * size + free_space_start]
+        puts "Free space start == %04x" % [(offset * size) + free_space_start]
         c = cursor(free_space_start).backward
         zero_bytes = 0
         zero_bytes += 1 while c.read_uint8.zero?

data/lib/innodb/page/inode.rb

@@ -83,7 +83,7 @@ module Innodb
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_list_entry,

data/lib/innodb/page/sys_data_dictionary_header.rb

@@ -68,7 +68,7 @@ module Innodb
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_data_dictionary_header,

data/lib/innodb/page/sys_ibuf_header.rb

@@ -27,7 +27,7 @@ module Innodb
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_ibuf_header,

data/lib/innodb/page/sys_rseg_header.rb

@@ -66,7 +66,7 @@ module Innodb
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_rseg_header,

data/lib/innodb/page/trx_sys.rb

@@ -182,7 +182,7 @@ module Innodb
       def each_region(&block)
         return enum_for(:each_region) unless block_given?
 
-        super(&block)
+        super
 
         yield Region.new(
           offset: pos_trx_sys_header,

data/lib/innodb/page.rb

@@ -101,11 +101,8 @@ module Innodb
     # Initialize a page by passing in a buffer containing the raw page contents.
     # The buffer size should match the space's page size.
     def initialize(space, buffer, page_number = nil)
-      unless space && buffer
-        raise "Page can't be initialized from nil space or buffer (space: #{space}, buffer: #{buffer})"
-      end
-
-      raise "Buffer size #{buffer.size} is different than space page size" unless space.page_size == buffer.size
+      raise "Page can't be initialized from nil space or buffer (space: #{space}, buffer: #{buffer})" unless buffer
+      raise "Buffer size #{buffer.size} is different than space page size" if space && space.page_size != buffer.size
 
       @space  = space
       @buffer = buffer
@@ -141,7 +138,7 @@ module Innodb
     # return value of the block.
     def cursor(buffer_offset)
       new_cursor = BufferCursor.new(@buffer, buffer_offset)
-      new_cursor.push_name("space[#{space.name}]")
+      new_cursor.push_name("space[#{space&.name || 'unknown'}]")
       new_cursor.push_name("page[#{name}]")
 
       if block_given?
@@ -205,74 +202,137 @@ module Innodb
       ALLOCATED: {
         value: 0,
         description: "Freshly allocated",
-        usage: "page type field has not been initialized",
       },
       UNDO_LOG: {
         value: 2,
         description: "Undo log",
-        usage: "stores previous values of modified records",
       },
       INODE: {
         value: 3,
         description: "File segment inode",
-        usage: "bookkeeping for file segments",
       },
       IBUF_FREE_LIST: {
         value: 4,
         description: "Insert buffer free list",
-        usage: "bookkeeping for insert buffer free space management",
       },
       IBUF_BITMAP: {
         value: 5,
         description: "Insert buffer bitmap",
-        usage: "bookkeeping for insert buffer writes to be merged",
       },
       SYS: {
         value: 6,
         description: "System internal",
-        usage: "used for various purposes in the system tablespace",
       },
       TRX_SYS: {
         value: 7,
         description: "Transaction system header",
-        usage: "bookkeeping for the transaction system in system tablespace",
       },
       FSP_HDR: {
         value: 8,
         description: "File space header",
-        usage: "header page (page 0) for each tablespace file",
       },
       XDES: {
         value: 9,
         description: "Extent descriptor",
-        usage: "header page for subsequent blocks of 16,384 pages",
       },
       BLOB: {
         value: 10,
         description: "Uncompressed BLOB",
-        usage: "externally-stored uncompressed BLOB column data",
       },
       ZBLOB: {
         value: 11,
         description: "First compressed BLOB",
-        usage: "externally-stored compressed BLOB column data, first page",
       },
       ZBLOB2: {
         value: 12,
         description: "Subsequent compressed BLOB",
-        usage: "externally-stored compressed BLOB column data, subsequent page",
+      },
+      UNKNOWN: {
+        value: 13,
+        description: "Unknown",
+      },
+      COMPRESSED: {
+        value: 14,
+        description: "Compressed",
+      },
+      ENCRYPTED: {
+        value: 15,
+        description: "Encrypted",
+      },
+      COMPRESSED_AND_ENCRYPTED: {
+        value: 16,
+        description: "Compressed and Encrypted",
+      },
+      ENCRYPTED_RTREE: {
+        value: 17,
+        description: "Encrypted R-tree",
+      },
+      SDI_BLOB: {
+        value: 18,
+        description: "Uncompressed SDI BLOB",
+      },
+      SDI_ZBLOB: {
+        value: 19,
+        description: "Compressed SDI BLOB",
+      },
+      LEGACY_DBLWR: {
+        value: 20,
+        description: "Legacy doublewrite buffer",
+      },
+      RSEG_ARRAY: {
+        value: 21,
+        description: "Rollback Segment Array",
+      },
+      LOB_INDEX: {
+        value: 22,
+        description: "Index of uncompressed LOB",
+      },
+      LOB_DATA: {
+        value: 23,
+        description: "Data of uncompressed LOB",
+      },
+      LOB_FIRST: {
+        value: 24,
+        description: "First page of an uncompressed LOB",
+      },
+      ZLOB_FIRST: {
+        value: 25,
+        description: "First page of a compressed LOB",
+      },
+      ZLOB_DATA: {
+        value: 26,
+        description: "Data of compressed LOB",
+      },
+      ZLOB_INDEX: {
+        value: 27,
+        description: "Index of compressed LOB",
+      },
+      ZLOB_FRAG: {
+        value: 28,
+        description: "Fragment of compressed LOB",
+      },
+      ZLOB_FRAG_ENTRY: {
+        value: 29,
+        description: "Index of fragment for compressed LOB",
+      },
+      SDI: {
+        value: 17_853,
+        description: "Serialized Dictionary Information",
+      },
+      RTREE: {
+        value: 17_854,
+        description: "R-tree index",
       },
       INDEX: {
         value: 17_855,
         description: "B+Tree index",
-        usage: "table and index data stored in B+Tree structure",
       },
     }.freeze
 
     PAGE_TYPE_BY_VALUE = PAGE_TYPE.each_with_object({}) { |(k, v), h| h[v[:value]] = k }
 
     # A page number representing "undefined" values, (4294967295).
-    UNDEFINED_PAGE_NUMBER = 2**32 - 1
+    UNDEFINED_PAGE_NUMBER = (2**32) - 1
 
     # A helper to check if a page number is the undefined page number.
     def self.undefined?(page_number)
@@ -285,6 +345,10 @@ module Innodb
       page_number unless undefined?(page_number)
     end
 
+    def self.page_type_by_value(value)
+      PAGE_TYPE_BY_VALUE[value] || value
+    end
+
     # Return the "fil" header from the page, which is common for all page types.
     def fil_header
       @fil_header ||= cursor(pos_fil_header).name("fil_header") do |c|
@@ -294,7 +358,7 @@ module Innodb
           prev: c.name("prev") { Innodb::Page.maybe_undefined(c.read_uint32) },
           next: c.name("next") { Innodb::Page.maybe_undefined(c.read_uint32) },
           lsn: c.name("lsn") { c.read_uint64 },
-          type: c.name("type") { PAGE_TYPE_BY_VALUE[c.read_uint16] },
+          type: c.name("type") { Innodb::Page.page_type_by_value(c.read_uint16) },
           flush_lsn: c.name("flush_lsn") { c.read_uint64 },
           space_id: c.name("space_id") { c.read_uint32 }
         )
@@ -405,7 +469,7 @@ module Innodb
 
     # Is the page in the doublewrite buffer?
     def in_doublewrite_buffer?
-      space&.system_space? && space&.doublewrite_page?(offset)
+      space&.system_space? && space.doublewrite_page?(offset)
     end
 
     # Is the space ID stored in the header different from that of the space

data/lib/innodb/record.rb

@@ -39,6 +39,16 @@ module Innodb
       row&.map { |r| "%s=%s" % [r.name, r.value.inspect] }&.join(", ")
     end
 
+    def full_value_with_externs_for_field(field)
+      blob_value = field.value
+      extern_page = field.extern && page.space.page(field.extern.page_number)
+      while extern_page
+        blob_value += extern_page.blob_data
+        extern_page = extern_page.next_blob_page
+      end
+      blob_value
+    end
+
     def undo
       return nil unless roll_pointer
       return unless (innodb_system = @page.space.innodb_system)

data/lib/innodb/record_describer.rb

@@ -119,16 +119,15 @@ module Innodb
 
     def generate_class(name = "Describer_#{object_id}")
       str = "class #{name}\n".dup
-      str << "  type %s\n" % [
-        description[:type].inspect,
-      ]
+      str << format("  type %s\n", description[:type].inspect)
       %i[key row].each do |group|
         description[group].each do |item|
-          str << "  %s %s, %s\n" % [
+          str << format(
+            "  %s %s, %s\n",
             group,
             item[:name].inspect,
-            item[:type].map(&:inspect).join(", "),
-          ]
+            item[:type].map(&:inspect).join(", ")
+          )
         end
       end
       str << "end\n"

data/lib/innodb/space.rb

@@ -276,6 +276,12 @@ module Innodb
       fsp[:space_id]
     end
 
+    def checked_page_class!(page, expected_class)
+      return page if page.instance_of?(expected_class)
+
+      raise "Page #{page.offset} is not the correct type, found: #{page.class}, expected: #{expected_class}"
+    end
+
     # Return the page number for the space's TRX_SYS page.
     def page_trx_sys
       5
@@ -283,13 +289,15 @@ module Innodb
 
     # Get the Innodb::Page::TrxSys page for a system space.
     def trx_sys
-      page(page_trx_sys) if system_space?
+      raise "Transaction System is only available in system spaces" unless system_space?
+
+      checked_page_class!(page(page_trx_sys), Innodb::Page::TrxSys)
     end
 
     def rseg_page?(page_number)
       return false unless trx_sys
 
-      !trx_sys.rsegs.select { |rseg| rseg.space_id.zero? && rseg.page_number == page_number }.empty?
+      trx_sys.rsegs.any? { |rseg| rseg.space_id.zero? && rseg.page_number == page_number }
     end
 
     # Return the page number for the space's SYS data dictionary header.
@@ -299,7 +307,9 @@ module Innodb
 
     # Get the Innodb::Page::SysDataDictionaryHeader page for a system space.
     def data_dictionary_page
-      page(page_sys_data_dictionary) if system_space?
+      raise "Data Dictionary is only available in system spaces" unless system_space?
+
+      checked_page_class!(page(page_sys_data_dictionary), Innodb::Page::SysDataDictionaryHeader)
     end
 
     # Get an Innodb::List object for a specific list by list name.
@@ -327,7 +337,7 @@ module Innodb
         # for IBD files, if they haven't added indexes online.
         (3...@pages).each do |page_number|
           page = page(page_number)
-          yield page_number if page.type == :INDEX && page.root?
+          yield page_number if page.is_a?(Innodb::Page::Index) && page.root?
         end
       end
 
@@ -358,7 +368,7 @@ module Innodb
     def each_inode(&block)
       return enum_for(:each_inode) unless block_given?
 
-      each_inode_list.each do |_name, list|
+      each_inode_list do |_name, list|
         list.each do |page|
           page.each_allocated_inode(&block)
         end

data/lib/innodb/undo_record.rb

@@ -238,7 +238,7 @@ module Innodb
     end
 
     def row_string
-      row&.reject(&:nil?)&.map { |r| r && "%s=%s" % [r[:name], r[:value].inspect] }&.join(", ")
+      row&.compact&.map { |r| r && format("%s=%s", r[:name], r[:value].inspect) }&.join(", ")
     end
 
     def string

data/lib/innodb/util/buffer_cursor.rb

@@ -76,7 +76,7 @@ class BufferCursor
   # position, raw byte buffer, and array of names.
   def print_trace(_cursor, position, bytes, name)
     slice_size = 16
-    bytes.each_slice(slice_size).each_with_index do |slice_bytes, slice_count|
+    bytes.each_slice(slice_size).with_index do |slice_bytes, slice_count|
       @trace_io.puts "%06i %s %-32s  %s" % [
         position + (slice_count * slice_size),
         direction == :backward ? "←" : "→",

data/lib/innodb/util/hex_format.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module HexFormat
+  LINE_SIZE = 16
+  GROUP_SIZE = 8
+  GROUP_FORMAT_LENGTH = ((LINE_SIZE.to_f / GROUP_SIZE).ceil * (GROUP_SIZE * 3))
+
+  def self.format_group(data)
+    data.map { |n| "%02x" % n.ord }.join(" ")
+  end
+
+  def self.format_groups(data, size)
+    data.each_slice(size).map { |g| format_group(g) }.join("  ")
+  end
+
+  def self.format_printable(data)
+    data.join.gsub(/[^[:print:]]/, ".")
+  end
+
+  def self.format_hex(data)
+    data.chars.each_slice(LINE_SIZE).with_index do |bytes, i|
+      yield format("%08i  %-#{GROUP_FORMAT_LENGTH}s  |%-#{LINE_SIZE}s|",
+                   (i * LINE_SIZE), format_groups(bytes, GROUP_SIZE), format_printable(bytes))
+    end
+
+    nil
+  end
+
+  def self.puts(data, io: $stdout)
+    format_hex(data) { |line| io.puts(line) }
+  end
+end

data/lib/innodb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Innodb
-  VERSION = "0.12.0"
+  VERSION = "0.14.0"
 end

data/lib/innodb/xdes.rb

@@ -118,7 +118,7 @@ module Innodb
     def each_page_status
       return enum_for(:each_page_status) unless block_given?
 
-      bitmap.each_byte.each_with_index do |byte, byte_index|
+      bitmap.each_byte.with_index do |byte, byte_index|
         (0..3).each do |page_offset|
           page_number = start_page + (byte_index * 4) + page_offset
           page_bits = ((byte >> (page_offset * BITS_PER_PAGE)) & BITMAP_BV_ALL)

data/lib/innodb.rb

@@ -14,10 +14,10 @@ module Innodb
   end
 end
 
-require "pp"
 require "digest/crc32c"
 require "innodb/util/buffer_cursor"
 require "innodb/util/read_bits_at_offset"
+require "innodb/util/hex_format"
 
 require "innodb/version"
 require "innodb/stats"

metadata

@@ -1,16 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: innodb_ruby
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Jeremy Cole
 - Davi Arnaut
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-12 00:00:00.000000000 Z
+date: 2024-11-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.8
 - !ruby/object:Gem::Dependency
   name: bindata
   requirement: !ruby/object:Gem::Requirement
@@ -51,6 +65,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.1
+- !ruby/object:Gem::Dependency
+  name: getoptlong
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: histogram
   requirement: !ruby/object:Gem::Requirement
@@ -85,14 +113,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.10.0
+        version: 3.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.10.0
+        version: 3.11.0
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -174,14 +202,16 @@ files:
 - lib/innodb/undo_log.rb
 - lib/innodb/undo_record.rb
 - lib/innodb/util/buffer_cursor.rb
+- lib/innodb/util/hex_format.rb
 - lib/innodb/util/read_bits_at_offset.rb
 - lib/innodb/version.rb
 - lib/innodb/xdes.rb
 homepage: https://github.com/jeremycole/innodb_ruby
 licenses:
 - BSD-3-Clause
-metadata: {}
-post_install_message: 
+metadata:
+  rubygems_mfa_required: 'false'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -196,8 +226,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.5.16
+signing_key:
 specification_version: 4
 summary: InnoDB data file parser
 test_files: []