innodb_ruby 0.8.1 → 0.8.5

data/lib/innodb/data_type.rb

@@ -0,0 +1,360 @@
+# -*- encoding : utf-8 -*-
+
+require "stringio"
+require "bigdecimal"
+require "date"
+
+class Innodb::DataType
+
+  # MySQL's Bit-Value Type (BIT).
+  class BitType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      nbits = modifiers.fetch(0, 1)
+      raise "Unsupported width for BIT type." unless nbits >= 0 and nbits <= 64
+      @width = (nbits + 7) / 8
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def value(data)
+      "0b%b" % BinData::const_get("Uint%dbe" % (@width * 8)).read(data)
+    end
+  end
+
+  class IntegerType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = base_type_width_map[base_type]
+      @unsigned = properties.include?(:UNSIGNED)
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def base_type_width_map
+      {
+        :BOOL       => 1,
+        :BOOLEAN    => 1,
+        :TINYINT    => 1,
+        :SMALLINT   => 2,
+        :MEDIUMINT  => 3,
+        :INT        => 4,
+        :INT6       => 6,
+        :BIGINT     => 8,
+      }
+    end
+
+    def value(data)
+      nbits = @width * 8
+      @unsigned ? get_uint(data, nbits) : get_int(data, nbits)
+    end
+
+    def get_uint(data, nbits)
+      BinData::const_get("Uint%dbe" % nbits).read(data)
+    end
+
+    def get_int(data, nbits)
+      BinData::const_get("Int%dbe" % nbits).read(data) ^ (-1 << (nbits - 1))
+    end
+  end
+
+  class FloatType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = 4
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    # Read a little-endian single-precision floating-point number.
+    def value(data)
+      BinData::FloatLe.read(data)
+    end
+  end
+
+  class DoubleType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = 8
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    # Read a little-endian double-precision floating-point number.
+    def value(data)
+      BinData::DoubleLe.read(data)
+    end
+  end
+
+  # MySQL's Fixed-Point Type (DECIMAL), stored in InnoDB as a binary string.
+  class DecimalType
+    attr_reader :name, :width
+
+    # The value is stored as a sequence of signed big-endian integers, each
+    # representing up to 9 digits of the integral and fractional parts. The
+    # first integer of the integral part and/or the last integer of the
+    # fractional part might be compressed (or packed) and are of variable
+    # length. The remaining integers (if any) are uncompressed and 32 bits
+    # wide.
+    MAX_DIGITS_PER_INTEGER = 9
+    BYTES_PER_DIGIT = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4]
+
+    def initialize(base_type, modifiers, properties)
+      precision, scale = sanity_check(modifiers)
+      integral = precision - scale
+      @uncomp_integral = integral / MAX_DIGITS_PER_INTEGER
+      @uncomp_fractional = scale / MAX_DIGITS_PER_INTEGER
+      @comp_integral = integral - (@uncomp_integral * MAX_DIGITS_PER_INTEGER)
+      @comp_fractional = scale - (@uncomp_fractional * MAX_DIGITS_PER_INTEGER)
+      @width = @uncomp_integral * 4 + BYTES_PER_DIGIT[@comp_integral] +
+               @comp_fractional * 4 + BYTES_PER_DIGIT[@comp_fractional]
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def value(data)
+      # Strings representing the integral and fractional parts.
+      intg, frac = "", ""
+
+      stream = StringIO.new(data)
+      mask = sign_mask(stream)
+
+      intg << get_digits(stream, mask, @comp_integral)
+
+      (1 .. @uncomp_integral).each do
+        intg << get_digits(stream, mask, MAX_DIGITS_PER_INTEGER)
+      end
+
+      (1 .. @uncomp_fractional).each do
+        frac << get_digits(stream, mask, MAX_DIGITS_PER_INTEGER)
+      end
+
+      frac << get_digits(stream, mask, @comp_fractional)
+
+      # Convert to something resembling a string representation.
+      str = mask.to_s.chop + intg + '.' + frac
+
+      BigDecimal.new(str).to_s('F')
+    end
+
+    private
+
+    # Ensure width specification (if any) is compliant.
+    def sanity_check(modifiers)
+      raise "Invalid width specification" unless modifiers.size <= 2
+      precision = modifiers.fetch(0, 10)
+      raise "Unsupported precision for DECIMAL type" unless
+        precision >= 1 and precision <= 65
+      scale = modifiers.fetch(1, 0)
+      raise "Unsupported scale for DECIMAL type" unless
+        scale >= 0 and scale <= 30 and scale <= precision
+      [precision, scale]
+    end
+
+    # The sign is encoded in the high bit of the first byte/digit. The byte
+    # might be part of a larger integer, so apply the bit-flipper and push
+    # back the byte into the stream.
+    def sign_mask(stream)
+      byte = BinData::Uint8.read(stream)
+      sign = byte & 0x80
+      byte.assign(byte ^ 0x80)
+      stream.rewind
+      byte.write(stream)
+      stream.rewind
+      (sign == 0) ? -1 : 0
+    end
+
+    # Return a string representing an integer with a specific number of digits.
+    def get_digits(stream, mask, digits)
+      nbits = BYTES_PER_DIGIT[digits] * 8
+      return "" unless nbits > 0
+      value = (BinData::const_get("Int%dbe" % nbits).read(stream) ^ mask)
+      # Preserve leading zeros.
+      ("%0" + digits.to_s + "d") % value
+    end
+  end
+
+  # Fixed-length character type.
+  class CharacterType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = modifiers.fetch(0, 1)
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+  end
+
+  class VariableCharacterType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = modifiers[0]
+      raise "Invalid width specification" unless modifiers.size == 1
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def value(data)
+      # The SQL standard defines that VARCHAR fields should have end-spaces
+      # stripped off.
+      data.sub(/[ ]+$/, "")
+    end
+  end
+
+  # Fixed-length binary type.
+  class BinaryType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = modifiers.fetch(0, 1)
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+  end
+
+  class VariableBinaryType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = modifiers[0]
+      raise "Invalid width specification" unless modifiers.size == 1
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+  end
+
+  class BlobType
+    attr_reader :name
+
+    def initialize(base_type, modifiers, properties)
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+  end
+
+  class YearType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = 1
+      @display_width = modifiers.fetch(0, 4)
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def value(data)
+      year = BinData::Uint8.read(data)
+      return (year % 100).to_s if @display_width != 4
+      return (year + 1900).to_s if year != 0
+      "0000"
+    end
+  end
+
+  class TimeType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = 3
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def value(data)
+      time = BinData::Int24be.read(data) ^ (-1 << 23)
+      sign = "-" if time < 0
+      time = time.abs
+      "%s%02d:%02d:%02d" % [sign, time / 10000, (time / 100) % 100, time % 100]
+    end
+  end
+
+  class DateType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = 3
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def value(data)
+      date = BinData::Int24be.read(data) ^ (-1 << 23)
+      day = date & 0x1f
+      month = (date >> 5) & 0xf
+      year = date >> 9
+      "%04d-%02d-%02d" % [year, month, day]
+    end
+  end
+
+  class DatetimeType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = 8
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    def value(data)
+      datetime = BinData::Int64be.read(data) ^ (-1 << 63)
+      date = datetime / 1000000
+      year, month, day = [date / 10000, (date / 100) % 100, date % 100]
+      time = datetime - (date * 1000000)
+      hour, min, sec = [time / 10000, (time / 100) % 100, time % 100]
+      "%04d-%02d-%02d %02d:%02d:%02d" % [year, month, day, hour, min, sec]
+    end
+  end
+
+  class TimestampType
+    attr_reader :name, :width
+
+    def initialize(base_type, modifiers, properties)
+      @width = 4
+      @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+    end
+
+    # Returns the UTC timestamp as a value in 'YYYY-MM-DD HH:MM:SS' format.
+    def value(data)
+      timestamp = BinData::Uint32be.read(data)
+      return "0000-00-00 00:00:00" if timestamp.zero?
+      DateTime.strptime(timestamp.to_s, '%s').strftime "%Y-%m-%d %H:%M:%S"
+    end
+  end
+
+  # Maps base type to data type class.
+  TYPES = {
+    :BIT        => BitType,
+    :BOOL       => IntegerType,
+    :BOOLEAN    => IntegerType,
+    :TINYINT    => IntegerType,
+    :SMALLINT   => IntegerType,
+    :MEDIUMINT  => IntegerType,
+    :INT        => IntegerType,
+    :INT6       => IntegerType,
+    :BIGINT     => IntegerType,
+    :FLOAT      => FloatType,
+    :DOUBLE     => DoubleType,
+    :DECIMAL    => DecimalType,
+    :NUMERIC    => DecimalType,
+    :CHAR       => CharacterType,
+    :VARCHAR    => VariableCharacterType,
+    :BINARY     => BinaryType,
+    :VARBINARY  => VariableBinaryType,
+    :TINYBLOB   => BlobType,
+    :BLOB       => BlobType,
+    :MEDIUMBLOB => BlobType,
+    :LONGBLOB   => BlobType,
+    :TINYTEXT   => BlobType,
+    :TEXT       => BlobType,
+    :MEDIUMTEXT => BlobType,
+    :LONGTEXT   => BlobType,
+    :YEAR       => YearType,
+    :TIME       => TimeType,
+    :DATE       => DateType,
+    :DATETIME   => DatetimeType,
+    :TIMESTAMP  => TimestampType,
+  }
+
+  def self.make_name(base_type, modifiers, properties)
+    name = base_type.to_s
+    name << '(' + modifiers.join(',') + ')' if not modifiers.empty?
+    name << " "
+    name << properties.join(' ')
+    name.strip
+  end
+
+  def self.new(base_type, modifiers, properties)
+    raise "Data type '#{base_type}' is not supported" unless TYPES.key?(base_type)
+    TYPES[base_type].new(base_type, modifiers, properties)
+  end
+end

data/lib/innodb/field.rb

@@ -1,23 +1,32 @@
 # -*- encoding : utf-8 -*-
-require "innodb/field_type"
+
+require "innodb/data_type"
 
 # A single field in an InnoDB record (within an INDEX page). This class
 # provides essential information to parse records, including the length
 # of the fixed-width and variable-width portion of the field.
 class Innodb::Field
-  attr_reader :position, :type
+  attr_reader :position, :name, :data_type, :nullable
 
   # Size of a reference to data stored externally to the page.
   EXTERN_FIELD_SIZE = 20
 
-  def initialize(position, data_type, *properties)
+  def initialize(position, name, type_definition, *properties)
     @position = position
-    @type = Innodb::FieldType.new(data_type.to_s, properties)
+    @name = name
+    @nullable = properties.delete(:NOT_NULL) ? false : true
+    base_type, modifiers = parse_type_definition(type_definition.to_s)
+    @data_type = Innodb::DataType.new(base_type, modifiers, properties)
+  end
+
+  # Return whether this field can be NULL.
+  def nullable?
+    @nullable
   end
 
   # Return whether this field is NULL.
   def null?(record)
-    type.nullable? && record[:header][:field_nulls][position]
+    nullable? && record[:header][:field_nulls][position]
   end
 
   # Return whether a part of this field is stored externally (off-page).
@@ -25,25 +34,71 @@ class Innodb::Field
     record[:header][:field_externs][position]
   end
 
+  def variable?
+    @data_type.is_a? Innodb::DataType::BlobType or
+    @data_type.is_a? Innodb::DataType::VariableBinaryType or
+    @data_type.is_a? Innodb::DataType::VariableCharacterType
+  end
+
+  def blob?
+    @data_type.is_a? Innodb::DataType::BlobType
+  end
+
   # Return the actual length of this variable-length field.
   def length(record)
-    if type.variable?
+    if variable?
       len = record[:header][:field_lengths][position]
     else
-      len = type.length
+      len = @data_type.width
     end
     extern?(record) ? len - EXTERN_FIELD_SIZE : len
   end
 
   # Read an InnoDB encoded data field.
   def read(record, cursor)
+    cursor.name(@data_type.name) { cursor.get_bytes(length(record)) }
+  end
+
+  # Read the data value (e.g. encoded in the data).
+  def value(record, cursor)
     return :NULL if null?(record)
-    cursor.name(type.name) { type.reader.read(cursor, length(record)) }
+    data = read(record, cursor)
+    @data_type.respond_to?(:value) ? @data_type.value(data) : data
   end
 
   # Read an InnoDB external pointer field.
-  def read_extern(record, cursor)
+  def extern(record, cursor)
     return nil if not extern?(record)
-    cursor.name(type.name) { type.reader.read_extern(cursor) }
+    cursor.name(@name) { read_extern(cursor) }
+  end
+
+  private
+
+  # Return an external reference field. An extern field contains the page
+  # address and the length of the externally stored part of the record data.
+  def get_extern_reference(cursor)
+    {
+      :space_id     => cursor.name("space_id")    { cursor.get_uint32 },
+      :page_number  => cursor.name("page_number") { cursor.get_uint32 },
+      :offset       => cursor.name("offset")      { cursor.get_uint32 },
+      :length       => cursor.name("length")      { cursor.get_uint64 & 0x3fffffff }
+    }
+  end
+
+  def read_extern(cursor)
+    cursor.name("extern") { get_extern_reference(cursor) }
+  end
+
+  # Parse a data type definition and extract the base type and any modifiers.
+  def parse_type_definition(type_string)
+    if matches = /^([a-zA-Z0-9]+)(\(([0-9, ]+)\))?$/.match(type_string)
+      base_type = matches[1].upcase.to_sym
+      if matches[3]
+        modifiers = matches[3].sub(/[ ]/, "").split(/,/).map { |s| s.to_i }
+      else
+        modifiers = []
+      end
+      [base_type, modifiers]
+    end
   end
 end

data/lib/innodb/fseg_entry.rb

@@ -1,6 +1,8 @@
 # -*- encoding : utf-8 -*-
+
 # An InnoDB file segment entry, which appears in a few places, such as the
 # FSEG header of INDEX pages, and in the TRX_SYS pages.
+
 class Innodb::FsegEntry
   # The size (in bytes) of an FSEG entry, which contains a two 32-bit integers
   # and a 16-bit integer.

data/lib/innodb/index.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+
 # An InnoDB index B-tree, given an Innodb::Space and a root page number.
 class Innodb::Index
   attr_reader :root
@@ -97,6 +98,10 @@ class Innodb::Index
     @root.fseg_header[name]
   end
 
+  def field_names
+    record_describer.field_names
+  end
+
   # Iterate through all file segments in the index.
   def each_fseg
     unless block_given?

data/lib/innodb/inode.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+
 class Innodb::Inode
   # The number of "slots" (each representing one page) in the fragment array
   # within each Inode entry.

data/lib/innodb/list.rb

@@ -1,7 +1,9 @@
 # -*- encoding : utf-8 -*-
+
 # An abstract InnoDB "free list" or FLST (renamed to just "list" here as it
 # frequently is used for structures that aren't free lists). This class must
 # be sub-classed to provide an appropriate #object_from_address method.
+
 class Innodb::List
   # An "address", which consists of a page number and byte offset within the
   # page. This points to the list "node" pointers (prev and next) of the

data/lib/innodb/log.rb

@@ -1,5 +1,7 @@
 # -*- encoding : utf-8 -*-
+
 # An InnoDB transaction log file.
+
 class Innodb::Log
   HEADER_SIZE   = 4 * Innodb::LogBlock::BLOCK_SIZE
   HEADER_START  = 0

data/lib/innodb/log_block.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+
 require "innodb/cursor"
 require "pp"
 

data/lib/innodb/log_group.rb

@@ -0,0 +1,79 @@
+# -*- encoding : utf-8 -*-
+
+class Innodb::LogGroup
+  def initialize
+    @files = []
+  end
+
+  def add_log(file)
+    if file = Innodb::Log.new(file)
+      @files.push file
+    else
+      raise "Couldn't open #{file}"
+    end
+  end
+
+  def each_block
+    @files.each do |file|
+      file.each_block do |block_number, block|
+        yield block_number, block
+      end
+    end
+  end
+
+  def current_tail_position
+    max       = 0
+    max_file  = nil
+    max_block = nil
+
+    @files.each_with_index do |file, file_number|
+      file.each_block do |block_number, block|
+        if block.header[:block] > max
+          max   = block.header[:block]
+          max_file  = file_number
+          max_block = block_number
+        end
+      end
+    end
+
+    { :file => max_file, :block => max_block }
+  end
+
+  def successor_position(position)
+    if position[:block] == @files[position[:file]].blocks
+      if position[:file] == @files.size
+        { :file => 0, :block => 0 }
+      else
+        { :file => position[:file] + 1, :block => 0 }
+      end
+    else
+      { :file => position[:file], :block => position[:block] + 1 }
+    end
+  end
+
+  def block(file_number, block_number)
+    @files[file_number].block(block_number)
+  end
+
+  def block_if_newer(old_block, new_block)
+    return new_block if old_block.nil?
+    #puts "old: #{old_block.header[:block]} new: #{new_block.header[:block]}"
+    if new_block.header[:block] >= old_block.header[:block]
+      new_block
+    end
+  end
+
+  def tail_blocks
+    position = current_tail_position
+    current_block = nil
+    while true
+      until block_if_newer(current_block, new_block = block(position[:file], position[:block]))
+        #puts "Waiting at the tail: #{position[:file]} #{position[:block]}"
+        sleep 0.1
+      end
+      yield new_block
+      position = successor_position(position)
+      current_block = new_block
+    end
+  end
+end

data/lib/innodb/page/fsp_hdr_xdes.rb

@@ -1,4 +1,5 @@
 # -*- encoding : utf-8 -*-
+
 require "innodb/list"
 require "innodb/xdes"