manageiq-smartstate 0.1.0

data/lib/metadata/ScanProfile/modules/HostScanItemFile.rb

@@ -0,0 +1,51 @@
+require 'metadata/linux/LinuxUtils'
+
+module HostScanItemFile
+  def parse_data(ssu, data, &_blk)
+    if data.nil?
+      d = scan_definition
+
+      st = Time.now
+      $log.info "Scanning [Profile-Files] information."
+      yield({:msg => 'Scanning Profile-File'}) if block_given?
+
+      fs_files = d["stats"].collect { |s| s["target"] }.uniq
+
+      $log.info "Retrieving file metadata for targets."
+      files = ssu.shell_exec("ls -lLd --full-time #{fs_files.join(' ')} 2>/dev/null; true").split("\n")
+      files = MiqLinux::Utils.parse_ls_l_fulltime(files)
+      files.each do |f|
+        f.delete(:hard_links)
+        f[:rsc_type] = f.delete(:ftype)
+        f[:contents] = nil
+      end
+
+      md5_files = files.collect { |f| f[:name] if f[:rsc_type] == 'file' }.compact
+      $log.info "Retrieving md5 values for targets."
+      md5_files = ssu.shell_exec("md5sum #{md5_files.join(' ')} 2>/dev/null; true").split("\n")
+      md5_files.each do |line|
+        parts = line.chomp.split(' ')
+        md5, fname = parts[0], parts[1..-1].join(' ')
+        file = files.find { |f| f[:name] == fname }
+        file[:md5] = md5 unless file.nil?
+      end
+
+      cat_files = d["stats"].collect { |s| s["target"] if s["content"] }.compact.uniq
+      unless cat_files.empty?
+        $log.info "Retrieving content for specified targets."
+        cat_files = ssu.shell_exec("ls -1d #{cat_files.join(' ')} 2>/dev/null; true").split("\n")
+        cat_files.each do |fname|
+          fname = fname.chomp
+          next if fname.empty?
+          file = files.find { |f| f[:name] == fname }
+          $log.debug "Retrieving content for #{fname}."
+          file[:contents] = ssu.shell_exec("cat #{fname} 2>/dev/null; true")
+        end
+      end
+
+      d[:data] = files
+
+      $log.info "Scanning [Profile-Files] information ran for [#{Time.now - st}] seconds."
+    end
+  end
+end

data/lib/metadata/ScanProfile/modules/HostScanItemNteventlog.rb

@@ -0,0 +1,84 @@
+require 'metadata/util/event_log_filter'
+
+require 'digest/md5'
+
+module HostScanItemNteventlog
+  def parse_data(vim, data, &_blk)
+    d = scan_definition
+    filter = d["content"][0][:filter]
+    if filter.nil?
+      $log.warn("MIQ(#{self.class.name}.parse_data) Unable to find hostd filter in scan profile [host default]")
+      return
+    end
+
+    data = vim.browseDiagnosticLogEx('hostd') if data.nil?
+    d[:data] = hostd_log_to_hashes(data, filter)
+  end
+
+  #
+  # Diagnostic Log Parsing
+  #
+
+  def hostd_log_to_hashes(log, filter = nil)
+    filter ||= {}
+    EventLogFilter.prepare_filter!(filter)
+
+    ret = []
+    rec_count = 0
+
+    log['lineText'].each do |line|
+      next unless line[0, 1] == '['
+      next unless i = line.index(']')
+      header, message = line[1..i - 1], line[i + 1..-1]
+
+      # Process the header parts
+      parts = header.split(' ')
+      next unless parts.length >= 5
+
+      # Different versions of ESX hosts order the log messages differently
+      #   old: date, time, quoted source, hash, level
+      #   new: date, time, hash, level, quoted source
+      # Check that the last character of the last part is a quote to determine
+      #   the versions
+      if parts[-1][-1, 1] != "'"
+        level = -1
+        source_start = 2
+        source_end = -3
+      else
+        level = 3
+        source_start = 4
+        source_end = -1
+      end
+
+      level = parts[level]
+      level = "warn" if level == "warning"
+      next if EventLogFilter.filter_by_level?(level, filter)
+
+      source = parts[source_start..source_end].join(' ')
+      source = source[1..-2] # Remove the surrounding quotes
+      next if EventLogFilter.filter_by_source?(source, filter)
+
+      generated = Time.parse(parts[0..1].join(' ')).utc.iso8601
+      next if EventLogFilter.filter_by_generated?(generated, filter)
+
+      message.strip!
+      next if EventLogFilter.filter_by_message?(message, filter)
+
+      name = 'hostd'
+
+      ret << {
+        :generated => generated,
+        :name      => name,
+        :level     => level,
+        :source    => source,
+        :message   => message,
+        :uid       => Digest::MD5.hexdigest("#{generated} #{name} #{level} #{source} #{message}")
+      }
+
+      rec_count += 1
+      break if EventLogFilter.filter_by_rec_count?(rec_count, filter)
+    end
+
+    ret
+  end
+end

data/lib/metadata/ScanProfile/modules/VmScanItemFile.rb

@@ -0,0 +1,39 @@
+module VmScanItemFile
+  def to_xml
+    xml = @xml_class.newNode("scan_item")
+    xml.add_attributes("guid" => @params["guid"], "name" => @params["name"], "item_type" => @params["item_type"])
+
+    scan_definition['stats'].each do |d|
+      if d["data"]
+        xml_partial = xml.root.add_element("filesystem")
+        xml_partial.add_attributes(d["data"].root.attributes)
+        xml_partial.add_attribute("id", d["target"])
+        e = d["data"].root.elements[1]
+        until e.blank?
+          xml_partial << e
+          e = d["data"].root.elements[1]
+        end
+      end
+    end
+
+    xml
+  end
+
+  def parse_data(vm, data, &_blk)
+    if data.nil?
+      st = Time.now
+      $log.info "Scanning [Profile-Files] information."
+      yield({:msg => 'Scanning Profile-File'}) if block_given?
+      scan_definition["stats"].each do |d|
+        # MD5deep scanning will raise an error if the path does not exist.
+        begin
+          # Skip if we already have data for this element
+          options = {'contents' => d['content']}
+          d["data"] = MD5deep.scan_glob(vm.rootTrees[0], d["target"], options) if d["data"].nil?
+        rescue
+        end
+      end
+      $log.info "Scanning [Profile-Files] information ran for [#{Time.now - st}] seconds."
+    end
+  end
+end

data/lib/metadata/ScanProfile/modules/VmScanItemNteventlog.rb

@@ -0,0 +1,34 @@
+module VmScanItemNteventlog
+  def to_xml
+    xml = @xml_class.newNode('scan_item')
+    xml.add_attributes('guid' => @params['guid'], 'name' => @params['name'], 'item_type' => @params['item_type'])
+    d = scan_definition
+    xml.root << d[:data] if d[:data]
+    xml
+  end
+
+  def parse_data(vm, data, &_blk)
+    if data.nil?
+      d = scan_definition
+      if d[:data].nil?
+        if vm.rootTrees[0].guestOS == "Windows"
+          begin
+            st = Time.now
+            $log.info "Scanning [Profile-EventLogs] information."
+            yield({:msg => 'Scanning Profile-EventLogs'}) if block_given?
+            ntevent = Win32EventLog.new(vm.rootTrees[0])
+            ntevent.readAllLogs(d['content'])
+          rescue MiqException::NtEventLogFormat
+            $log.warn $!.to_s
+          rescue => err
+            $log.error "Win32EventLog: #{err}"
+            $log.error err.backtrace.join("\n")
+          ensure
+            d[:data] = ntevent.xmlDoc
+            $log.info "Scanning [Profile-EventLogs] information ran for [#{Time.now - st}] seconds."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metadata/ScanProfile/modules/VmScanItemRegistry.rb

@@ -0,0 +1,64 @@
+module VmScanItemRegistry
+  def to_xml
+    xml = @xml_class.newNode("scan_item")
+    xml.add_attributes("guid" => @params["guid"], "name" => @params["name"], "item_type" => @params["item_type"])
+
+    scan_definition["content"].each do |d|
+      if d[:data]
+        xml_partial = xml.root.add_element("registry")
+        xml_partial.add_attributes("base_path" => base_reg_path(d), "id" => build_reg_path(d))
+        xml_partial << d[:data]
+      end
+    end
+    xml
+  end
+
+  def parse_data(_vm, data)
+    if data
+      scan_definition["content"].each do |d|
+        d[:data] = MIQRexml.findRegElement(build_reg_path(d), data.root) if d[:data].nil?
+      end
+    end
+  end
+
+  def build_reg_path(scanHash)
+    path = get_long_hive_name(scanHash["hive"]) + "\\" + scanHash["key"]
+
+    # Include the value as part of the search if we have a valid value
+    path += "\\" + scanHash["value"] if include_value?(scanHash)
+
+    path
+  end
+
+  def base_reg_path(scanHash)
+    path = File.join(scanHash["hive"], scanHash["key"])
+    path.tr!("/", "\\")
+    # If we are not processing the value the base path is the hive + key, minus the last path element
+    path = path.split("\\")[0..-2].join("\\") unless include_value?(scanHash)
+    path
+  end
+
+  def include_value?(scanHash)
+    # Include the value as part of the search if we have a valid value
+    value = scanHash["value"].to_s.strip
+    value = "" if value.length == 1 && value[0, 1] == "*"
+    if !value.blank?
+      return true
+    else
+      return false
+    end
+  end
+
+  def get_long_hive_name(hive)
+    case hive
+    when "HKLM"
+      "HKEY_LOCAL_MACHINE"
+    when "HKCU"
+      "HKEY_CURRENT_USER"
+    when "HKCR"
+      "HKEY_CLASSES_ROOT"
+    else
+      hive
+    end
+  end
+end

data/lib/metadata/VMMount/VMMount.rb

@@ -0,0 +1,81 @@
+require 'ostruct'
+require 'disk/MiqDisk'
+require 'fs/MiqFS/MiqFS'
+require 'metadata/VMMount/VMPlatformMount'
+require 'metadata/util/win32/system_path_win'   # Used to check for Windows system partition
+
+class VMMount
+  attr_reader :disk
+
+  def initialize(filename, ost = nil)
+    $log.debug "Initializing VMMount, filename: #{filename}" if $log
+    @mountDrive = nil
+    @diskInfo = OpenStruct.new
+    @ost = OpenStruct.new unless ost
+    @disk = nil
+    @fs = nil
+
+    # Use the MIQ functions to load virtual disk image
+    # Check if there is a flat file and redirect to the proper name
+    # TODO: This function should be improved to open the current file and check for the
+    #       redirection.  Right now it just looks for the expected flat filename.
+    $log.debug "vmmount called for disk image: [#{filename}]"
+    @diskInfo.fileName = getMountFile(filename)
+
+    @platMount = VMPlatformMount.new(@diskInfo, @ost)
+    @platMount.preMount
+    begin
+      @disk = MiqDisk.getDisk(@diskInfo)
+      raise "Failed to open disk: #{@diskInfo.fileName}" if @disk.nil?
+
+      @parts = @disk.getPartitions
+      raise "No Partitions found on disk: #{@diskInfo.fileName}" if @parts.nil?
+
+      $log.debug "VMMount mounting partitions of #{filename}" if $log
+      @parts.each do |p|
+        fsPartitions = MiqFS.getFS(p)
+        unless fsPartitions.nil?
+          # Always set the fs pointer to the first partition so we return something.
+          @fs = fsPartitions if @fs.nil?
+
+          # If the systemRoot does not throw an error then we found it
+          # and should return this fs.  Otherwise, continue looking.
+          begin
+            Win32::SystemPath.systemRoot(fsPartitions)
+            @fs = fsPartitions
+            break
+          rescue
+            # $log.warn "No Windows partition here."
+          end
+        end
+      end
+      $log.debug "vmmount complete for: [#{filename}]"
+    rescue => err
+      @platMount.postMount
+      raise err
+    end
+  end
+
+  def getMountFile(fn)
+    file_miq = File.join(File.dirname(fn), File.basename(fn, ".*") + ".miq")
+    return file_miq if File.exist?(file_miq)
+    fn
+  end
+
+  def unMountImage
+    if @disk
+      @disk.close
+      @platMount.postMount
+    end
+  end
+
+  def getMountDrive
+    return @fs if @fs
+    @mountDrive
+  end
+
+  def mounted?
+    return true unless getMountDrive.nil?
+    false
+  end
+end

data/lib/metadata/VMMount/VMPlatformMount.rb

@@ -0,0 +1,18 @@
+require 'sys-uname'
+
+class VMPlatformMount
+  def initialize(dInfo, ost)
+    $log.debug "Initializing VMPlatformMount" if $log
+    @dInfo = dInfo
+    @ost = ost
+
+    if Sys::Platform::OS == :windows
+      require "metadata/VMMount/VMPlatformMountWin"
+      extend VMPlatformMountWin
+    else
+      require "metadata/VMMount/VMPlatformMountLinux"
+      extend VMPlatformMountLinux
+    end
+    init
+  end # def initialize
+end # class VMPlatformMount

data/lib/metadata/VMMount/VMPlatformMountLinux.rb

@@ -0,0 +1,75 @@
+require 'util/miq-password'
+
+module VMPlatformMountLinux
+  VCBNAME     = '/usr/sbin/vcbVmName'
+  VCBSNAPSHOT = '/usr/sbin/vcbSnapshot'
+
+  def init
+    $log.debug "Initializing VMPlatformMountLinux: #{@dInfo.fileName}" if $log
+    @ems = nil
+    @snapSsId = nil
+
+    unless @ost.force
+      $log.debug "Initializing VMPlatformMountLinux: force flag = false" if $log
+      return
+    end
+
+    return unless $miqHostCfg
+    return unless $miqHostCfg.emsLocal
+    return unless File.exist? VCBNAME
+    return unless File.exist? VCBSNAPSHOT
+    return unless File.extname(@dInfo.fileName) == ".vmdk"
+
+    $log.debug "VMPlatformMountLinux::init: emsLocal = #{$miqHostCfg.emsLocal}" if $log
+    @ems = $miqHostCfg.ems[$miqHostCfg.emsLocal]
+    #
+    # TODO: This won't always yield the VM name.
+    #       While it works most of the time, we need to find a better way to do this.
+    #
+    @vmName  = File.basename(File.dirname(@dInfo.fileName))
+    $log.debug "VMPlatformMountLinux::init: vmName = \"" # {@vmName}\"" if $log
+    #
+    # TODO: passing the user name and password on the command line is BAD.
+    #       Change this to use web-services.
+    #
+    @cs = MiqPassword::Key.new
+    cmd = "#{VCBNAME} -h #{@ems['host']} -u #{@ems['user']} -p #{@cs.decrypt64(@ems['password'])} -s \"Name:#{@vmName}\" | grep \"moref:\""
+    #
+    # XXX Don't forget to remove this. We're logging the username and password, not good.
+    #
+    $log.debug "VMPlatformMountLinux::init: cmd = #{cmd}" if $log
+    @vmMoref = `#{cmd}`.chomp
+    $log.debug "VMPlatformMountLinux::init: vmMoref = \"#{@vmMoref}\"" if $log
+  end
+
+  def preMount
+    return unless @ost.force
+    return unless @ems
+    cmd = "#{VCBSNAPSHOT} -h #{@ems['host']} -u #{@ems['user']} -p #{@cs.decrypt64(@ems['password'])} -c \"#{@vmMoref}\" \"miq#{@vmName}snap\" | grep \"SsId:\""
+    #
+    # XXX Don't forget to remove this. We're logging the username and password, not good.
+    #
+    $log.debug "VMPlatformMountLinux::preMount: cmd = #{cmd}" if $log
+    @snapSsId = `#{cmd}`.chomp
+    $log.debug "VMPlatformMountLinux::preMount: snapSsId = \"#{@snapSsId}\"" if $log
+    # The VMMount code selects the disk file before the snapshot is taken.
+    # The file it selects should be the parent of the snapshot, so we should be OK.
+    # This may not be the case fot other types of VMs.
+    # @dInfo.baseOnly = true
+  end
+
+  def postMount
+    return unless @ost.force
+    #
+    # TODO: Ensure the snapshot gets removed in the event something happens between the preMount and postMount.
+    #
+    return unless @snapSsId
+    cmd = "#{VCBSNAPSHOT} -h #{@ems['host']} -u #{@ems['user']} -p #{@cs.decrypt64(@ems['password'])} -d \"#{@vmMoref}\" \"#{@snapSsId}\""
+    #
+    # XXX Don't forget to remove this. We're logging the username and password, not good.
+    #
+    $log.debug "VMPlatformMountLinux::postMount: cmd = #{cmd}" if $log
+    result = `#{cmd}`.chomp
+    $log.debug "VMPlatformMountLinux::postMount: result = #{result}" if $log
+  end
+end # module VMPlatformMountLinux

data/lib/metadata/VMMount/VMPlatformMountWin.rb

@@ -0,0 +1,13 @@
+module VMPlatformMountWin
+  def init
+    true
+  end
+
+  def preMount
+    true
+  end
+
+  def postMount
+    true
+  end
+end # module VMPlatformMountWin

data/lib/metadata/VmConfig/GetNativeCfg.rb

@@ -0,0 +1,45 @@
+require 'util/miq-xml'
+require 'util/runcmd'
+require 'metadata/VmConfig/VmConfig'
+
+class GetNativeCfg
+  LSHW = "lshw"
+
+  def self.new
+    lshwXml = MiqUtil.runcmd("#{LSHW} -xml")
+    nodeHash = Hash.new { |h, k| h[k] = [] }
+    doc = MiqXml.load(lshwXml)
+    doc.find_match("//node").each { |n| nodeHash[n.attributes["id"].split(':', 2)[0]] << n }
+
+    hardware = ""
+
+    nodeHash["disk"].each do |d|
+      diskid = d.find_first('businfo').get_text.to_s
+      next unless diskid
+      sn = d.find_first('size')
+      # If there's no size node, assume it's a removable drive.
+      next unless sn
+      busType, busAddr = diskid.split('@', 2)
+      if busType == "scsi"
+        f1, f2 = busAddr.split(':', 2)
+        f2 = f2.split('.')[1]
+        busAddr = "#{f1}:#{f2}"
+      else
+        busAddr['.'] = ':'
+      end
+      diskid = busType + busAddr
+      filename = d.find_first('logicalname').get_text.to_s
+      hardware += "#{diskid}.present = \"TRUE\"\n"
+      hardware += "#{diskid}.filename = \"#{filename}\"\n"
+    end
+
+    VmConfig.new(hardware)
+  end
+end
+
+if __FILE__ == $0
+
+  cfg = GetNativeCfg.new
+  cfg.getDiskFileHash.each { |dtag, df| puts "#{dtag}\t=> #{df}" }
+
+end