manageiq-smartstate 0.1.0

data/lib/db/MiqBdb/MiqBdbBtree.rb

@@ -0,0 +1,219 @@
+require 'binary_struct'
+require 'util/miq-hash_struct'
+
+require_relative 'MiqBdbPage'
+
+module MiqBerkeleyDB
+  class MiqBdbBtreeDatabase
+    BTREE_HEADER = BinaryStruct.new([
+      'L',    'unused1',      # 72-75: Unused space.
+      'L',    'minkey',       # 76-79: Btree: Minkey
+      'L',    're_rlen',      # 80-83: Recno: fixed-len record length
+      'L',    're_pad',       # 84-87: Recno: fixed-len record pad
+      'L',    'root',         # 88-91: Root page
+      'a368', 'unused2',      # 92-459: Unused space
+      'L',    'crypto_magic', # 460-463: Crypto magic number.
+      'a12',  'trash',        # 464-475: Trash space - Do not use.
+      'a16',  'iv',           # 476-495: Crypto IV.
+      'a20',  'chksum',       # 496-511: Page chksum.
+    ])
+
+    SIZEOF_BTREE_HEADER = BTREE_HEADER.size
+
+    BINTERNAL_HEADER = BinaryStruct.new([
+      'S',    'len',          # 00-01: Key/data item length
+      'C',    'ptype',        #    02: Page type (AND DELETE FLAG)
+      'C',    'unused',       #    03: Padding, unused
+      'L',    'pgno',         # 04-07: Page number of referenced page
+      'L',    'nrecs',        # 08-11: Subtree record count
+    ])
+
+    SIZEOF_BINTERNAL_HEADER = BINTERNAL_HEADER.size
+
+    #
+    # Each index references a group of bytes on the page
+    #
+    B_KEYDATA   = 1 # Key/data item
+    B_DUPLICATE = 2 # Duplicate key/data item
+    B_OVERFLOW  = 3 # Overflow key/data item
+
+    B_DELETE    = 0x80
+
+    OFFSET_LEN  = 2
+
+    #
+    # The btree levels are numbered from the leaf to the root, starting
+    # with 1, so the leaf is level 1, its parent is level 2, and so on.
+    # We maintain this level on all btree pages, but the only place that
+    # we actually need it is on the root page.  It would not be difficult
+    # to hide the byte on the root page once it becomes an internal page,
+    # so we could get this byte back if we needed it for something else.
+    #
+
+    attr_reader   :bdb, :header
+
+    def initialize(bdb)
+      # Read pointer is positioned to hash header.
+      @bdb = bdb
+      @header = BTREE_HEADER.decode(@bdb.read(SIZEOF_BTREE_HEADER))
+    end
+
+    def close
+      @bdb = @header = nil
+    end
+
+    def keys(page)
+      leaves(page) do |leaf|
+        btree_leaf_keys(leaf) do |k|
+          yield k
+        end
+      end
+    end
+
+    def values(page)
+      leaves(page) do |leaf|
+        btree_leaf_values(leaf) do |v|
+          yield v
+        end
+      end
+    end
+
+    def pairs(page)
+      leaves(page) do |leaf|
+        btree_leaf(leaf) do |k, v|
+          yield k, v
+        end
+      end
+    end
+
+    def pages
+      pagenum = @header['root']
+      while page = MiqBdbPage.getPage(self, pagenum)
+
+        yield page
+
+        pagenum = page.next
+      end
+    end
+
+    def dump
+      out  = ""
+      out << "B-Tree Database Header\n"
+      out << "  minkey:          #{@header['minkey']}\n"
+      out << "  re_rlen:         #{@header['re_rlen']}\n"
+      out << "  re_pad:          #{@header['re_pad']}\n"
+      out << "  root:            #{@header['root']}\n"
+      out << "\n"
+      out
+    end
+
+    private
+
+    def btreei_dump(h, index = nil)
+      out  = ""
+      out << "B-Tree Internal Node"
+      out << " (#{index})" if index
+      out << "\n"
+      out << "  key/data item length: #{h.len}\n"
+      out << "  page number:          #{h.pgno}\n"
+      out << "  page type:            #{MiqBdbPage.type2string(h.ptype)}\n"
+      out << "  subtree record count: #{h.nrecs}\n"
+      out << "\n"
+      out
+    end
+
+    def btree_leaf(page)
+      i = 0
+      while i < page.nentries
+        key   = entryData(i,   page)
+        value = entryData(i + 1, page)
+
+        yield key, value if key.size > 0
+        i += 2  # skip value
+      end
+    end
+
+    def btree_leaf_keys(page)
+      i = 0
+      while i < page.nentries
+        key   = entryData(i,   page)
+
+        yield key if key.size > 0
+        i += 2  # skip value
+      end
+    end
+
+    def btree_leaf_values(page)
+      i = 0
+      while i < page.nentries
+        key   = entryData(i,   page)
+        value = entryData(i + 1, page)
+
+        yield value if key.size > 0
+        i += 2  # skip value
+      end
+    end
+
+    def leaves(page)
+      for index in 0..page.nentries - 1
+        header = BINTERNAL_HEADER.decode(page.buf[entryOffset(index, page), (SIZEOF_BINTERNAL_HEADER)])
+
+        btpage = MiqBdbPage.getPage(self, header['pgno'])
+        bttype = MiqBdbPage.type2string(btpage.ptype)
+        case bttype
+        when "btree internal" then leaves(btpage) { |leaf| yield leaf }
+        when "btree leaf" then     yield btpage
+        else                   raise "Unexpected Page Type: #{bttype}"
+        end
+      end
+    end
+
+    def entryOffset(index, page)
+      i = index * OFFSET_LEN
+      page.data[i..i + 1].unpack('S')[0]
+    end
+
+    def entryType(index, page)
+      page.buf[entryOffset(index, page) + 2, 1].unpack('C')[0] & ~B_DELETE
+    end
+
+    def entryDataImmediate(index, page)
+      # B_KEYDATA
+      # 00-01: Key/data item length
+      #    02: Page type AND DELETE FLAG
+      #        Data Follows
+      len = page.buf[entryOffset(index, page), 2].unpack('S')[0]
+      page.buf[entryOffset(index, page) + 3, len].unpack("A#{len}")[0]
+    end
+
+    def entryDataOverflow(index, page)
+      # B_DUPLICATE and B_OVERFLOW
+      # 00-01: Padding, unused
+      #    02: Page type AND DELETE FLAG
+      #    03: Padding, unused
+      # 04-07: Next page number
+      # 08-11: Total length of item
+      pgno = page.buf[entryOffset(index, page) + 4, 4].unpack("S1")[0]
+      # Not using the following at the moment
+      # tlen = page.buf[entryOffset(index,page)+8,4].unpack("S1")[0]
+
+      data = ""
+      while pgno != 0
+        opage = MiqBdbPage.getPage(self, pgno)
+        data << opage.data[0, opage.offset]
+        pgno  = opage.next
+      end
+      data
+    end
+
+    def entryData(index, page)
+      type = entryType(index, page)
+      case type
+      when B_KEYDATA then   return entryDataImmediate(index, page)
+      when B_OVERFLOW then  return entryDataOverflow(index, page)
+      when B_DUPLICATE then raise "Unsupported Type: B_DUPLICATE"
+      else              raise "Unknown Type: #{type}"
+      end
+    end
+  end
+end

data/lib/db/MiqBdb/MiqBdbHash.rb

@@ -0,0 +1,199 @@
+require 'binary_struct'
+require 'util/miq-hash_struct'
+
+require_relative 'MiqBdbPage'
+
+module MiqBerkeleyDB
+  class MiqBdbHashDatabase
+    # Hash constants.
+    DB_HASH_DUP     = 0x01 # Duplicates.
+    DB_HASH_SUBDB   = 0x02 # Subdatabases.
+    DB_HASH_DUPSORT = 0x04 # Duplicates are sorted.
+
+    HASH_HEADER = BinaryStruct.new([
+      'L',    'max_bucket',   # 72-75: ID of Maximum bucket in use.
+      'L',    'high_mask',    # 76-79: Modulo mask into table.
+      'L',    'low_mask',     # 80-83: Modulo mask into table lower half.
+      'L',    'ffactor',      # 84-87: Fill factor.
+      'L',    'nelem',        # 88-91: Number of keys in hash table.
+      'L',    'h_charkey',    # 92-95: Value of hash(CHARKEY).
+      'a128', 'spares',       # 96-223: Spare pages for overflow.
+      'a236', 'unused',       # 224-459: Unused space.
+      'L',    'crypto_magic', # 460-463: Crypto magic number.
+      'a12',  'trash',        # 464-475: Trash space - Do not use.
+      'a16',  'iv',           # 476-495: Crypto IV.
+      'a20',  'chksum',       # 496-511: Page chksum.
+      # NOTE: There is a discrepency between the last two members.
+      # Offset notes show these as 20 and 16 bytes, when they are
+      # in fact 16 and 20 bytes respectively.
+    ])
+
+    SIZEOF_HASH_HEADER = HASH_HEADER.size
+
+    # Each index references a group of bytes on the page.
+    H_KEYDATA   = 1   # Key/data item.
+    H_DUPLICATE = 2   # Duplicate key/data item.
+    H_OFFPAGE   = 3   # Overflow key/data item.
+    H_OFFDUP    = 4   # Overflow page of duplicates.
+
+    OFFSET_LEN  = 2
+
+    attr_reader   :bdb, :header
+
+    def initialize(bdb)
+      # Read pointer is positioned to hash header.
+      @bdb = bdb
+      @header = HASH_HEADER.decode(@bdb.read(SIZEOF_HASH_HEADER))
+      @header['spares'] = @header['spares'].unpack('L*')
+    end
+
+    def close
+      @bdb = @header = nil
+    end
+
+    def npages
+      @bdb.npages
+    end
+
+    def nkeys
+      @header['nelem']
+    end
+
+    def pages
+      0.upto(@header['max_bucket']) do |b|
+        pagenum = bucket2page(b)
+        while page = MiqBdbPage.getPage(self, pagenum)
+          yield page
+
+          pagenum = page.next
+        end
+      end
+    end
+
+    def keys(page)
+      i = 0
+      while i < page.nentries
+        key  = entryData(i,   page)
+
+        yield key
+        i += 2  # skip value
+      end
+    end
+
+    def values(page)
+      i = 0
+      while i < page.nentries
+        value = entryData(i + 1, page)
+
+        yield value
+        i += 2  # skip key
+      end
+    end
+
+    def pairs(page)
+      i = 0
+      while i < page.nentries
+        key  = entryData(i,   page)
+        value = entryData(i + 1, page)
+
+        yield key, value
+        i += 2
+      end
+    end
+
+    def dump
+      out  = ""
+      out << "Hash Database Header\n"
+      out << "  nkeys:           #{@header['nelem']}\n"
+      out << "  high_mask:       0x#{'%01x' % @header['high_mask']}\n"
+      out << "  low_mask:        0x#{'%01x' % @header['low_mask']}\n"
+      out << "  ffactor:         #{@header['ffactor']}\n"
+      out << "  h_charkey:       0x#{'%08x' % @header['h_charkey']}\n"
+
+      out << "  spare points:    "
+      @header['spares'].each { |s| out << s.to_s << " " }
+      out << "\n"
+
+      out << "\n"
+      out
+    end
+
+    private
+
+    #
+    # The spares table indicates the page number at which each doubling begins.
+    # From this page number we subtract the number of buckets already allocated
+    # so that we can do a simple addition to calculate the page number here.
+    #
+
+    def bucket2page(bucket)
+      bucket + @header['spares'][MiqBdbHashDatabase.log2(bucket + 1)]
+    end
+
+    def self.log2(num)
+      limit = 1
+      i = 0
+      while limit < num
+        limit <<= 1
+        i += 1
+      end
+
+      i
+    end
+
+    def entryDataOverflow(_index, offset, page)
+      pgno = page.buf[offset + 4, 4].unpack("S1")[0]
+      # Not using the following at the moment
+      # tlen = page.buf[offset + 8, 4].unpack("S1")[0]
+
+      data = ""
+      while pgno != 0
+        opage = MiqBdbPage.getPage(self, pgno)
+        data << opage.data[0, opage.offset]
+        pgno  = opage.next
+      end
+      data
+    end
+
+    def entryDataImmediate(index, offset, page)
+      len  = entryLen(index, offset, page)
+      data = page.buf[offset + 1, len].unpack("A#{len - 1}")[0]
+      data
+    end
+
+    def entryData(index, page)
+      # $log.debug "Getting index #{index} (#{index + 1} / #{page.nentries}) for page #{page.current} (#{page.current + 1} / #{npages})"
+
+      type, offset = entryTypeAndOffset(index, page)
+      data = case type
+             when H_KEYDATA then   entryDataImmediate(index, offset, page)
+             when H_OFFPAGE then   entryDataOverflow(index, offset, page)
+             when H_DUPLICATE then raise "Unsupported Type: H_DUPLICATE"
+             when H_OFFDUP then    raise "Unsupported Type: H_OFFDUP"
+             else                  raise "Unknown Type: #{type.inspect}"
+             end
+
+      data
+    end
+
+    def entryTypeAndOffset(index, page)
+      offset = entryOffset(index, page)
+      type = entryType(offset, page)
+      return type, offset
+    end
+
+    def entryType(offset, page)
+      page.buf[offset, 1].ord
+    end
+
+    def entryOffset(index, page)
+      i = index * OFFSET_LEN
+      page.data[i, 2].unpack('S')[0]
+    end
+
+    def entryLen(index, offset, page)
+      item_end = index == 0 ? page.pagesize : entryOffset(index - 1, page)
+      (item_end - offset)
+    end
+  end
+end

data/lib/db/MiqBdb/MiqBdbPage.rb

@@ -0,0 +1,159 @@
+require 'enumerator'
+
+require 'binary_struct'
+require 'util/miq-hash_struct'
+
+module MiqBerkeleyDB
+  class MiqBdbPage
+    # Page numbers.
+    PGNO_INVALID = 0
+    PGNO_BASE_MD = 0
+
+    # Page types
+    P_INVALID       = 0   # Invalid page type
+    P_DUPLICATE     = 1   # Duplicate. DEPRECATED in 3.1
+    P_HASH          = 2   # Hash
+    P_IBTREE        = 3   # Btree internal
+    P_IRECNO        = 4   # Recno internal
+    P_LBTREE        = 5   # Btree leaf
+    P_LRECNO        = 6   # Recno leaf
+    P_OVERFLOW      = 7   # Overflow
+    P_HASHMETA      = 8   # Hash metadata page
+    P_BTREEMETA     = 9   # Btree metadata page
+    P_QAMMETA       = 10  # Queue metadata page
+    P_QAMDATA       = 11  # Queue data page
+    P_LDUP          = 12  # Off-page duplicate leaf
+
+    def self.type2string(t)
+      case t
+      when P_INVALID then   "invalid"
+      when P_DUPLICATE then "duplicate (deprecated)"
+      when P_HASH then      "hash"
+      when P_IBTREE then    "btree internal"
+      when P_IRECNO then    "recno internal"
+      when P_LBTREE then    "btree leaf"
+      when P_LRECNO then    "recno leaf"
+      when P_OVERFLOW then  "overflow"
+      when P_HASHMETA then  "hash metadata"
+      when P_BTREEMETA then "btree metadata"
+      when P_QAMMETA then   "queue metadata"
+      when P_QAMDATA then   "queue data"
+      when P_LDUP then      "offpage duplicate leaf"
+      else                  "unknown value of #{t}"
+      end
+    end
+
+    # Page Header.
+    HEADER = BinaryStruct.new([
+      # 00-07: Log sequence number (LSN)
+      'L', 'lsn_file',        #   0-3: LSN File
+      'L', 'lsn_offset',      #   4-7: LSN Offset
+      'L', 'pgno',            # 08-11: Current page number
+      'L', 'prev_pgno',       # 12-15: Previous page number
+      'L', 'next_pgno',       # 16-19: Next page number
+      'S', 'entries',         # 20-21: Number of items on the page
+      'S', 'hf_offset',       # 22-23: High free byte page offset
+      'C', 'level',           #    24: Btree tree level
+      'C', 'p_type'           #    25: Page type
+    ])
+
+    SIZEOF_HEADER = HEADER.size
+
+    #
+    # With many compilers sizeof(PAGE) == 28, while SIZEOF_PAGE == 26.
+    # We add in other things directly after the page header and need
+    # the SIZEOF_PAGE.  When giving the sizeof(), many compilers will
+    # pad it out to the next 4-byte boundary.
+    #
+    SIZEOF_PAGE = 26
+
+    attr_reader :header, :data, :pagesize, :buf
+
+    def self.getPage(db, pagenum)
+      return nil if pagenum == PGNO_INVALID
+      #     return nil if pagenum >= db.npages
+
+      buf = db.bdb.readPage(pagenum)
+      return nil if buf.nil? || buf[25, 1].ord == P_INVALID
+      MiqBdbPage.new(buf, db)
+    end
+
+    def initialize(buf, db)
+      raise "Nil buffer." if buf.nil?
+      @db         = db
+      @buf        = buf
+      @pagesize   = buf.size
+      @header     = HEADER.decode(@buf[0, SIZEOF_HEADER])
+      @data       = buf[SIZEOF_HEADER..-1]
+    end
+
+    def current
+      @header['pgno']
+    end
+
+    def next
+      @header['next_pgno']
+    end
+
+    def prev
+      @header['prev_pgno']
+    end
+
+    def nentries
+      @header['entries']
+    end
+
+    def offset
+      @header['hf_offset']
+    end
+
+    def level
+      @header['level']
+    end
+
+    def ptype
+      @header['p_type']
+    end
+
+    def keys
+      @db.keys(self) { |k| yield k }
+    end
+
+    def values
+      @db.values(self) { |v| yield v }
+    end
+
+    def pairs
+      @db.pairs(self) { |k, v| yield k, v }
+    end
+
+    # Dump page statistics like db_dump.
+    def dump
+      out  = ""
+      out << "Page #{current}\n"
+      out << "  type:            #{MiqBdbPage.type2string(ptype)}\n"
+      out << "  prev:            #{prev}\n"
+      out << "  next:            #{@header['next_pgno']}\n"
+      out << "  log seq num:     file=#{@header['lsn_file']}  offset=#{@header['lsn_offset']}\n"
+      out << "  level:           #{level}\n"
+
+      if @header['p_type'] == P_OVERFLOW
+        out << "  ref cnt:         #{nentries}\n"
+        out << "  len:             #{offset}\n"
+      else
+        out << "  entries:         #{nentries}\n"
+        out << "  offset:          #{offset}\n"
+      end
+      out << "  data size:       #{@data.size}\n"
+      out << "  data:            "
+
+      @data.bytes.take(20).each do |c|
+        out << sprintf("%.2x ", c)
+      end
+      out << "..." if @data.size > 20
+
+      out << "\n\n"
+      out
+    end
+  end
+end

data/lib/db/MiqBdb/MiqBdbUtil.rb

@@ -0,0 +1,18 @@
+require_relative 'MiqBdb'
+require_relative 'MiqBdbHash'
+require_relative 'MiqBdbPage'
+
+module MiqBerkeleyDB
+  class MiqBdbUtil
+    def initialize(fs = nil)
+      @fs = fs
+    end
+
+    def getkeys(fname)
+      bdb  = MiqBdb.new(fname, @fs)
+      keys = bdb.keys
+      bdb.close
+      keys
+    end
+  end
+end