manageiq-smartstate 0.1.0

data/lib/blackbox/VmBlackBox.rb

@@ -0,0 +1,103 @@
+require 'ostruct'
+require 'yaml'
+require 'util/miq-xml'
+require 'blackbox/xmlStorage'
+require 'metadata/VmConfig/VmConfig'
+require 'digest/md5'
+
+module Manageiq
+  class BlackBox
+    GLOBAL_CONFIG_FILE = "/miq.yml"
+
+    def initialize(vmName, ost = nil)
+      @config_name = vmName
+      @write_data_externally = true    # For now we are always writing externally
+
+      ost ||= OpenStruct.new
+
+      if ost.miqVm
+        @vmCfg = ost.miqVm.vm.vmConfig
+      elsif ost.skipConfig == true
+        @vmCfg = VmConfig.new({})
+      else
+        @vmCfg = VmConfig.new(@config_name)
+      end
+
+      # Get path to local data directory
+      if ost.config && ost.config.dataDir
+        @localDataDir = File.join(ost.config.dataDir, Digest::MD5.hexdigest(@config_name))
+      elsif $miqHostCfg && $miqHostCfg.dataDir
+        @localDataDir = File.join($miqHostCfg.dataDir, Digest::MD5.hexdigest(@config_name))
+      else
+        @localDataDir = "/tmp"
+      end
+
+      loadGlobalSettings
+
+      @xmlData = loadXmlConfig
+    end
+
+    def self.vmId(vmName)
+      Manageiq::BlackBox.new(vmName).vmId
+    end
+
+    def vmId=(uuid)
+      saveGlobalValue(:vmId, uuid)
+    end
+
+    def vmId
+      @cfg[:vmId]
+    end
+
+    def close
+    end
+
+    # This method is used to cleanup local storage data after it is sent to the server.
+    def postSync(options = {})
+      deleteLocalDataDir(options)
+    end
+
+    private
+
+    def loadGlobalSettings
+      @cfg = {:smart => false}
+      @cfg.merge!(YAML.load(readData(GLOBAL_CONFIG_FILE)))
+    rescue
+    end
+
+    def saveGlobalValue(key, value)
+      @cfg[key] = value
+      saveGlobalSettings
+    end
+
+    def saveGlobalSettings
+      x = ""
+      YAML.dump(@cfg, x)
+      writeData(GLOBAL_CONFIG_FILE, x)
+    end
+
+    def deleteLocalDataDir(options = {})
+      if @write_data_externally || options[:forceDelete]
+        if File.exist?(@localDataDir)
+          Dir.foreach(@localDataDir) { |f| File.delete(File.join(@localDataDir, f)) unless f[0..0] === "." }
+          Dir.delete(@localDataDir)
+        end
+      end
+    end
+
+    def writeData(filename, data)
+      Dir.mkdir(@localDataDir, 0755) unless File.exist?(@localDataDir)
+      filename2 = filename.tr("/", "_")
+      fullpath = File.join(@localDataDir, filename2)
+      f = File.open(fullpath, "w")
+      f.write(data.to_s)
+      f.close
+    end
+
+    def readData(filename)
+      filename2 = filename.tr("/", "_")
+      fullpath = File.join(@localDataDir, filename2)
+      File.read(fullpath)
+    end
+  end
+end

data/lib/blackbox/xmlStorage.rb

@@ -0,0 +1,180 @@
+module Manageiq
+  class BlackBox
+    METADATA_CONFIG_FILE = "/metadata/xmldata.yml"
+
+    def saveXmlData(xml, filename, options = {})
+      filename.downcase!
+      options = {:saveDiff => true}.merge(options)
+
+      xml.root.add_attribute("original_filename", filename)
+      scanTime = xml.root.attributes['created_on']
+      scanTime ||= xml.root.attributes[:created_on]
+      scanTime = scanTime.to_s
+      path = getXmlFileName(filename, scanTime)
+
+      xml_prev = nil
+      xml_prev = getLastXmlFile(filename) if options[:saveDiff]
+
+      xml = xml.to_xml if xml.kind_of?(Hash)  # XmlHash
+      if xml.kind_of?(Hash)  # XmlHash
+        writeData(path, xml.to_xml.to_s)
+        # TODO: Change to store xml ask Marshal dump file
+        # path = get_dump_name(path)
+        # data = Marshal.dump(xml)
+        # File.open(path, "wb") {|f| f.write(data)}
+      else
+        writeData(path, xml.to_s)
+      end
+      addConfigFile(filename, scanTime)
+
+      XmlDiffAndStore(filename, xml, xml_prev) if xml_prev
+    end
+
+    def get_dump_name(filename)
+      filepath = File.join(File.dirname(filename), "#{File.basename(filename, '.*')}.dmp")
+      filepath.tr!('\\', '/')
+      filepath
+    end
+
+    def loadXmlData(filename, ost = nil)
+      # Make sure we have a valid openstruct handle and "from_time" is in a valid format
+      ost = OpenStruct.new if ost.nil?
+      # Check 'from_time' value and possible remove from the open struct
+      validate_from_time(ost)
+      ret = OpenStruct.new(:items_selected => 0, :items_total => 0)
+
+      ret.xml = MiqXml.createDoc("<vmmetadata/>")
+      ret.xml.root.add_attributes("original_filename" => filename, "from_time" => ost.from_time.to_s, "taskid" => ost.taskid)
+      if filename == "vmevents"
+        # Change the name of the root element so the data does not go through state data processing.
+        ret.xml.root.name = "vmevents"
+        # ret.xml.root << @mk.view("events").find_range_by_hash(ost.from_time.nil? ? nil : {:timestamp=>ost.from_time}).to_xml.root
+      else
+        @xmlData.each do |x|
+          if x[:docs].include?(filename.downcase)
+            unless ret.last_scan
+              ret.last_scan = x[:time]
+            else
+              ret.last_diff_scan = x[:time] unless ret.last_diff_scan
+            end
+            ret.items_total += 1
+
+            # if we have a "from time" make sure we do not include anything older
+            next if ost.from_time && ost.from_time.to_i > x[:time].to_i
+
+            # load the xml and check what kind of xml we have (full or diff)
+            xmlNode = getXmlFile(filename, x[:time])
+            next if xmlNode.nil?
+            xmlNode = xmlNode.root
+
+            xml_type = getXmlType(xmlNode)
+
+            # If we have a "from_time" we are not sending full scans
+            next if ost.from_time && xml_type == "full"
+
+            # Create a new "item" element for the xml and record the scan type
+            e = ret.xml.root.add_element("item", {"scanType" => xml_type})
+            e << xmlNode
+            # Keep count of the number of items we add
+            ret.items_selected += 1
+          end
+        end
+      end
+      ret.xml.root.add_attributes("items_selected" => ret.items_selected, "items_total" => ret.items_total,
+                  "last_scan" => ret.last_scan, "last_diff_scan" => ret.last_diff_scan)
+      ret
+    end
+
+    private
+
+    def XmlDiffAndStore(filename, xml, xml_prev)
+      # Make sure we are processing to xml files in the proper order
+      if xml.root.attributes['created_on'] > xml_prev.root.attributes['created_on']
+        xml.extendXmlDiff
+        delta = xml.xmlDiff(xml_prev)
+        delta.root.add_attribute("original_filename", filename)
+        path = getXmlFileName(filename, xml_prev.root.attributes['created_on'])
+        writeData(path, delta.to_s)
+      end
+    end
+
+    def addConfigFile(filename, time)
+      item = findXmlConfigItem(time)
+      if item.nil?
+        # prepend new object to array
+        @xmlData.unshift(:time => time, :docs => [filename])
+        saveXmlConfig
+      else
+        # Update existing array item
+        unless item[:docs].include?(filename)
+          item[:docs] << filename
+          saveXmlConfig
+        end
+      end
+    end
+
+    def findXmlConfigItem(time)
+      item = nil
+      @xmlData.each do|x|
+        if x[:time] == time
+          item = x
+          break
+        end
+      end
+      item
+    end
+
+    def loadXmlConfig
+      data = readData(METADATA_CONFIG_FILE)
+      YAML.load(data)
+    rescue
+      []
+    end
+
+    def saveXmlConfig
+      writeData(METADATA_CONFIG_FILE, YAML.dump(@xmlData))
+    end
+
+    def getLastXmlFile(filename)
+      item = nil
+      # This is a sorted list with the newest items on top
+      @xmlData.each do|x|
+        if x[:docs].include?(filename)
+          item = x
+          break
+        end
+      end
+      return nil if item.nil?
+
+      xml = getXmlFile(filename, item[:time])
+      return nil if xml.nil?
+
+      type = getXmlType(xml)
+      return nil if type == "diff"
+      xml
+    end
+
+    def getXmlFile(filename, time)
+      MiqXml.load(readData(getXmlFileName(filename, time)))
+    rescue
+      # Return nil if we fail to read the xml file
+      nil
+    end
+
+    def getXmlFileName(filename, time)
+      timestr = time.to_s
+      File.join("/metadata", "#{timestr[0..7]}.#{timestr[8..11]}", filename + ".xml")
+    end
+
+    def getXmlType(xml)
+      return "diff" if xml.root.name.downcase == "xmldiff"
+      "full"
+    end
+
+    def validate_from_time(ost)
+      # Delete the "from_time" field
+      # TODO: do we still need to add from_time in the first place?
+      ost.delete_field("from_time") if ost.from_time
+    end
+  end
+end

data/lib/db/MiqBdb/MiqBdb.rb

@@ -0,0 +1,309 @@
+############################################################################################
+# Berkeley DB is an embedded database system that supports keyed access to data.
+# It stores information in one of the following ways.
+#
+#   B+tree:
+#      Stores keys in sorted order, using a default function that does lexicographical
+#      ordering of keys.
+#
+#   Hashing:
+#      Stores records in a hash table for fast searches based on strict equality,
+#      using a default that hashes on the key as a bit string.
+#      Extended Linear Hashing modifies the hash function used by the table as new
+#      records are inserted, in order to keep buckets underfull in the steady state.
+#
+#   Fixed and Variable-Length Records:
+#      Stores fixed- or variable-length records in sequential order. Record numbers
+#      may be immutable, requiring that new records be added only at the end of the
+#      database, or mutable, permitting new records to be inserted between existing
+#      records.
+#
+############################################################################################
+#
+#  This API mimics the Hash-Like Interface presented in the ruby-bdb project located at
+#  http://moulon.inra.fr/ruby/bdb.html
+#
+#  Class Methods
+#  =============
+#
+#  open the database
+#    open(filename)
+#
+#  Instance Methods
+#  ================
+#
+#  Iterators
+#
+#  Iterates over associations.  { |key, value| ... }
+###    each
+#
+#  Iterates over keys.  { |key| ... }
+###    each_key
+#
+#  Iterates over values. { |value| ... }
+###    each_value
+#
+#  Iterates over each duplicate associations for key.  { |key, value| ... }
+#    each_dup(key)
+#
+#  Iterates over each duplicate values for key.  { |value| ... }
+#    each_dup_value(key)
+#
+#  Returns true if the association from the key exists.
+#    has_key?(key)
+#
+#  Returns true if the association to the value exists.
+#    has_value?(value)
+#
+#  Returns true if the association from key is value
+#    has_both?(key, value)
+#
+#  Return database statistics.
+#    stat
+#
+#  Return the name of the file
+#    filename
+#
+#  Returns the array of the keys in the database
+###    keys
+#
+#  Returns the array of the values in the database.
+###    values
+#
+#  Return an array of all duplicate associations for key
+#    duplicates(key)
+#
+#  Return an array of all associations [key, value]
+#    to_a
+#
+#  Return an hash of all associations {key => value}
+#    to_hash
+#
+#  Returns the value corresponding the key
+#    self[key]
+#    get(key)
+#
+#  Returns the number of association in the database
+###    size
+#
+#
+############################################################################################
+
+require 'enumerator'
+
+require 'binary_struct'
+require 'util/miq-hash_struct'
+
+require_relative 'MiqBdbHash'
+require_relative 'MiqBdbBtree'
+
+module MiqBerkeleyDB
+  # BerkeleyDB constants.
+  DBMETASIZE     = 512        # Size of metadata (in bytes) on disk.
+  DB_HASHMAGIC   = 0x00061561 # Magic for Hash   database
+  DB_BTREEMAGIC  = 0x00053162 # Magic for BTree  database
+  DB_QAMMAGIC    = 0x00042253 # Magic for Queue  database
+  DB_LOGMAGIC    = 0x00040988 # Magic for Log    database
+  DB_RENAMEMAGIC = 0x00030800 # Magic for Rename database
+
+  # Database header.
+  DBHEADER = BinaryStruct.new([
+    'Q',   'lsn',          # 00-07: LSN.
+    'L',   'pgno',         # 08-11: Current page number.
+    'L',   'magic',        # 12-15: Magic number.
+    'L',   'version',      # 16-19: Version.
+    'L',   'pagesize',     # 20-23: Pagesize.
+    'C',   'encrypt_alg',  # 24: Encryption algorithm.
+    'C',   'p_type',       # 25: Page type.
+    'C',   'metaflags',    # 26: Meta-only flags.
+    'C',   'unused1',      # 27: Unused.
+    'L',   'free',         # 28-31: Free list page number.
+    'L',   'last_pgno',    # 32-35: Page number of last page in db.
+    'L',   'unused3',      # 36-39: Unused.
+    'L',   'key_count',    # 40-43: Cached key count.
+    'L',   'record_count', # 44-47: Cached record count.
+    'L',   'flags',        # 48-51: Flags: unique to each AM.
+    'a20', 'uid',          # 52-71: Unique file ID.
+  ])
+
+  SIZEOF_DBHEADER = DBHEADER.size
+
+  class MiqBdb
+    attr_reader :db, :header
+
+    def initialize(fileName = nil, fs = nil)
+      @fs = fs unless fs.nil?
+      open(fileName) unless fileName.nil?
+    end
+
+    # Open the database
+    def open(filename)
+      # Get header & check.
+      @filename = filename
+      @file = fileOpen(@filename)
+      @header = DBHEADER.decode(@file.read(SIZEOF_DBHEADER))
+
+      # Holds the page buf if we read all pages
+      @read_buf = nil
+
+      # Check header version - simple warning if less than 8.
+      if @header["version"] < 8
+        msg = "MiqBerkeleyDB: Database header version is less than 8."
+        if $log
+          $log.warn(msg)
+        else
+          puts "WARNING: " + msg
+        end
+      end
+
+      # We don't support encryption.
+      raise "MiqBerkeleyDB: This database uses encryption." if @header["encrypt_alg"] != 0
+
+      # Open db type (only hash for now, may expand to plugins in the future).
+      @db = case @header["magic"]
+            when DB_HASHMAGIC then   MiqBdbHashDatabase.new(self)
+            when DB_BTREEMAGIC then  MiqBdbBtreeDatabase.new(self)
+            when DB_QAMMAGIC then    raise "MiqBerkeleyDB: Database type is Queue"
+            when DB_LOGMAGIC then    raise "MiqBerkeleyDB: Database type is Log"
+            when DB_RENAMEMAGIC then raise "MiqBerkeleyDB: Database type is Rename"
+            else                     raise "MiqBerkeleyDB: Database type #{@header['magic']} is not supported"
+            end
+    end
+
+    # Close the database
+    def close
+      @db.close
+      @file.close
+      @db = @header = @file = @filename = nil
+    end
+
+    def read(len)
+      raise "File not open" if @file.nil?
+      @file.read(len)
+    end
+
+    def nbuckets
+      @db.header.max_bucket + 1
+    end
+
+    def nkeys
+      @header["key_count"]
+    end
+
+    def nrecs
+      @header["record_count"]
+    end
+
+    def npages
+      @header["last_pgno"] + 1
+    end
+
+    def pagesize
+      @header["pagesize"]
+    end
+
+    def pages
+      @db.pages { |page| yield page }
+    end
+
+    def readPage(pagenum, buf = @read_buf)
+      where = (pagenum * pagesize)
+      page = if buf.nil?
+               @file.seek(where)
+               @file.read(pagesize)
+             else
+               buf[where, pagesize]
+             end
+      page
+    end
+
+    def readAllPages
+      @file.seek(0)
+      @read_buf = @file.read(pagesize * npages)
+    end
+
+    def dump
+      out = "File Header\n"
+      out << "  lsn:             #{@header['lsn']}\n"
+      out << "  pgno:            #{@header['pgno']}\n"
+      out << "  magic:           0x#{'%08x' % @header['magic']}\n"
+      out << "  version:         #{@header['version']}\n"
+      out << "  pagesize:        #{@header['pagesize']}\n"
+      out << "  encrypt_alg:     #{@header['encrypt_alg']}\n"
+      out << "  page type:       #{MiqBdbPage.type2string(@header['p_type'])}\n"
+      out << "  metaflags:       0x#{'%08x' % @header['metaflags']}\n"
+      out << "  free:            #{@header['free']}\n"
+      out << "  last_pgno:       #{@header['last_pgno']}\n"
+      out << "  key_count:       #{@header['key_count']}\n"
+      out << "  record_count:    #{@header['record_count']}\n"
+      out << "  flags:           0x#{'%08x' % @header['flags']}\n"
+      out << "  uid:             #{uid_format}\n"
+      out << "\n"
+      out
+    end
+
+    def keys
+      keys = []
+      each_key { |k| keys << k }
+      keys
+    end
+
+    def values
+      values = []
+      each_value { |v| values << v }
+      values
+    end
+
+    def size
+      size = 0
+      each_key { |_k| size += 1 }
+      size
+    end
+
+    def each
+      @db.pages do |page|
+        page.pairs do |k, v|
+          yield k, v
+        end
+      end
+    end
+
+    def each_key
+      @db.pages do |page|
+        page.keys do |k|
+          yield k
+        end
+      end
+    end
+
+    def each_value
+      @db.pages do |page|
+        page.values do |v|
+          yield v
+        end
+      end
+    end
+
+    private
+
+    def fileSize
+      return File.size(@filename) if @fs.nil?
+      @fs.fileSize(@filename)
+    end
+
+    def fileOpen(fileName)
+      # Return a file object using MiqFS or File as the case may be.
+      if @fs.nil?
+        File.open(fileName, "rb")
+      else
+        @fs.fileOpen(fileName, "r")
+      end
+    end
+
+    def uid_format
+      out = ""
+      @header['uid'].each_byte { |b| out += sprintf("%02x ", b) }
+      out
+    end
+  end
+end