manageiq-smartstate 0.1.0

data/lib/fs/MiqFsUtil.rb

@@ -0,0 +1,383 @@
+require 'find'
+require 'fileutils'
+require 'yaml'
+require 'ostruct'
+require 'zlib'
+
+class MiqFsUtil
+  attr_accessor :verbose, :fromFs, :toFs
+
+  def initialize(fromFs, toFs, us = nil)
+    setUpdateSpec(us)
+    @fromFs = fromFs
+    @toFs = toFs
+
+    @verbose = false
+  end
+
+  def updateSpec=(us)
+    setUpdateSpec(us)
+  end
+
+  def setUpdateSpec(us)
+    if us
+      @csa = us
+      @csa = YAML.load_file(us) if @csa.kind_of? String
+      @csa = [@csa] if @csa.kind_of? Hash
+      raise "Invalid collection spec" unless @csa.kind_of? Array
+    else
+      @csa = nil
+    end
+  end
+
+  #
+  # Load the update spec from a file in the fromFs.
+  #
+  def loadUpdateSpec(path)
+    @fromFs.fileOpen(path) { |fo| @csa = YAML.load(fo.read) }
+  end
+
+  def update
+    raise "MiqFsUpdate.update: no current update spec" unless @csa
+    @csa.each { |cs| doUpdate(OpenStruct.new(cs)) }
+  end
+
+  def dumpSpec(specFile)
+    raise "MiqFsUpdate.dumpSpec: no current update spec" unless @csa
+    YAML.dump(@csa, File.open(specFile, "w"))
+  end
+
+  #
+  # Copy files and directories from fromFs to the toFs.
+  #
+  # FILE -> FILE
+  # FILE -> DIR
+  # DIR  -> DIR (recursive = true)
+  #
+  def copy(from, to, recursive = false)
+    allTargets = []
+    from = [from] unless from.kind_of?(Array)
+    from.each { |t| allTargets.concat(dirGlob(t)) }
+
+    raise "copy: no source files matched" if allTargets.length == 0
+    if allTargets.length > 1 || recursive
+      raise "copy: destination directory does not exist" unless @toFs.fileExists?(to)
+      raise "copy: destination must be a directory for multi-file copy" unless @toFs.fileDirectory?(to)
+    end
+
+    allTargets.each do |f|
+      owd = @fromFs.pwd
+      @fromFs.chdir(File.dirname(f))
+      f = File.basename(f)
+
+      #
+      # Copy plain files.
+      #
+      if @fromFs.fileFile?(f)
+        if @toFs.fileDirectory?(to)
+          tf = File.join(to, File.basename(f))
+        else
+          tf = to
+        end
+        copySingle(f, tf)
+        next
+      end
+
+      #
+      # If the recursive flag is not set, skip directories.
+      #
+      next unless recursive
+
+      #
+      # Recursively copy directory sub-tree.
+      #
+      @fromFs.chdir(f)
+      td = File.join(to, f)
+      @toFs.dirMkdir(td) unless @toFs.fileExists?(td)
+      @fromFs.findEach('.') do |ff|
+        tf = File.join(td, ff)
+        if @fromFs.fileDirectory?(ff)
+          @toFs.dirMkdir(tf)
+        elsif @fromFs.fileFile?(ff)
+          copySingle(ff, tf)
+        end
+      end # findEach
+      @fromFs.chdir(owd)
+    end # allTargets.each
+  end
+
+  private
+
+  def log_puts(str = "")
+    if $log
+      $log.info str
+    else
+      puts str
+    end
+  end
+
+  GLOB_CHARS = '*?[{'
+  def isGlob?(str)
+    str.count(GLOB_CHARS) != 0
+  end
+
+  def dirGlob(glb, *flags, &block)
+    return([glb]) unless isGlob?(glb)
+
+    if glb[0, 1] == '/'
+      dir = '/'
+      glb = glb[1..-1]
+    else
+      dir = @fromFs.pwd
+    end
+
+    matches = doGlob(glb.split('/'), dir, flags)
+    return(matches) unless block_given?
+
+    matches.each do |e|
+      block.call(e)
+    end
+    (false)
+  end
+
+  def doGlob(glbArr, dir, flags)
+    return [] if !glbArr || glbArr.length == 0
+
+    retArr = []
+    glb = glbArr[0]
+
+    dirForeach(dir) do |e|
+      if flags.length == 0
+        match = File.fnmatch(glb, e)
+      else
+        match = File.fnmatch(glb, e, flags)
+      end
+      if match
+        if glbArr.length == 1
+          retArr << File.join(dir, e)
+        else
+          next unless @fromFs.fileDirectory?(nf = File.join(dir, e))
+          retArr.concat(doGlob(glbArr[1..-1], nf, flags))
+        end
+      end
+    end
+    (retArr)
+  end
+
+  def copySingle(ff, tf)
+    if @verbose
+      log_puts "Copying: #{ff}"
+      log_puts "     to: #{tf}"
+    end
+    @fromFs.fileOpen(ff) do |ffo|
+      tfo = @toFs.fileOpen(tf, "wb")
+      while (buf = ffo.read(1024))
+        tfo.write(buf)
+      end
+      tfo.close
+    end
+  end
+
+  def doUpdate(cs)
+    #
+    # The directory where the collection will be created.
+    #
+    destDir = cs.todir
+
+    #
+    # Save the current directory.
+    #
+    fowd = @fromFs.pwd
+    towd = @toFs.pwd
+
+    begin
+      #
+      # The directory relative to which the files will be collected.
+      #
+      @fromFs.chdir(cs.basedir)
+
+      #
+      # Loop through the files and directories that are to be included in the collection.
+      #
+      cs.include.each do |i|
+        raise "File: #{i} does not exist" unless @fromFs.fileExists? i
+        #
+        # If this is a plain file, then include it in the collection.
+        #
+        unless @fromFs.fileDirectory? i
+          toFile = File.join(destDir, i)
+          makePath(File.dirname(toFile))
+          #
+          # If the file path matches an encrypt RE and doesn't
+          # match a noencrypt RE, then encrypt the contents of
+          # the file before copying it to the collection.
+          #
+          if cs.encrypt && cs.encrypt.detect { |e| i =~ e }
+            if !cs.noencrypt || !cs.noencrypt.detect { |ne| i =~ ne }
+              compressFile(i, toFile)
+              next
+            end
+          end
+          #
+          # If the file path matches an compress RE and doesn't
+          # match a nocompress RE, then compress the contents of
+          # the file before copying it to the collection.
+          #
+          if cs.compress && cs.compress.detect { |e| i =~ e }
+            if !cs.nocompress || !cs.nocompress.detect { |ne| i =~ ne }
+              compressFile(i, toFile)
+              next
+            end
+          end
+          copyFile(i, toFile)
+          next
+        end
+
+        #
+        # If this is a directory, then recursively copy its contents
+        # to the collection directory.
+        #
+        @fromFs.findEach(i) do |path|
+          #
+          # Prune directories that match an exclude RE.
+          #
+          if @fromFs.fileDirectory? path
+            @fromFs.findEachPrune if cs.exclude && cs.exclude.detect { |e| path =~ e }
+            next
+          end
+          #
+          # Skip files that match an exclude RE.
+          #
+          next if cs.exclude && cs.exclude.detect { |e| path =~ e }
+          toFile = File.join(destDir, path)
+          makePath(File.dirname(toFile))
+          #
+          # If the file path matches an encrypt RE and doesn't
+          # match a noencrypt RE, then encrypt the contents of
+          # the file before copying it to the collection.
+          #
+          if cs.encrypt && cs.encrypt.detect { |e| path =~ e }
+            if !cs.noencrypt || !cs.noencrypt.detect { |ne| path =~ ne }
+              compressFile(path, toFile)
+              next
+            end
+          end
+          #
+          # If the file path matches an compress RE and doesn't
+          # match a nocompress RE, then compress the contents of
+          # the file before copying it to the collection.
+          #
+          if cs.compress && cs.compress.detect { |e| path =~ e }
+            if !cs.nocompress || !cs.nocompress.detect { |ne| path =~ ne }
+              compressFile(path, toFile)
+              next
+            end
+          end
+          copyFile(path, toFile)
+        end
+      end if cs.include
+
+      #
+      # Remove files from the destination fs.
+      #
+      @toFs.chdir(destDir)
+      cs.remove.each do |r|
+        if r.kind_of? Regexp
+          #
+          # If the entry is a RE, then remove all files and directories in
+          # the destination directory that match the RE>
+          #
+          @toFs.findEach(".") do |p|
+            next unless r.match(p)
+
+            log_puts "\tRemoving: #{p}" if @verbose
+            if !@toFs.fileDirectory? p
+              @toFs.fileDelete(p)
+            else
+              @toFs.rmBranch(p)
+              @toFs.findEachPrune
+            end
+          end
+        else
+          #
+          # If the entry is a string, then it should be the path to the file
+          # or directory to be removed.
+          #
+          unless @toFs.exists? r
+            log_puts "Remove file: #{i} does not exist" if @verbose
+            next
+          end
+
+          log_puts "\tRemoving: #{r}" if @verbose
+          if !@toFs.fileDirectory? r
+            @toFs.fileDelete(r)
+          else
+            @toFs.rmBranch(r)
+          end
+        end
+      end if cs.remove
+    ensure
+      @fromFs.chdir(fowd)
+      @toFs.chdir(towd)
+    end
+  end
+
+  def makePath(path)
+    return if @toFs.fileExists? path
+    parentDir = @toFs.fileDirname(path)
+    makePath(parentDir) unless @toFs.fileExists? parentDir
+    @toFs.dirMkdir(path)
+  end
+
+  def copyFile(src, dest)
+    if @fromFs.respond_to?(:hasTag?) && @fromFs.hasTag?(src, "compressed")
+      decompressFile(src, dest)
+      return
+    end
+
+    if @verbose
+      log_puts "\t    COPY: #{src}"
+      log_puts "\t      TO: #{dest}"
+    end
+    @fromFs.fileOpen(src) do |ffo|
+      tfo = @toFs.fileOpen(dest, "wb")
+      while (buf = ffo.read(4096))
+        tfo.write(buf, buf.length)
+      end
+      tfo.close
+    end
+  end
+
+  def compressFile(src, dest)
+    if @verbose
+      log_puts "\tCOMPRESS: #{src}"
+      log_puts "\t      TO: #{dest}"
+    end
+    @fromFs.fileOpen(src) do |ffo|
+      tfo = @toFs.fileOpen(dest, "wb")
+      zipper = Zlib::Deflate.new
+      while (buf = ffo.read(4096))
+        zipper << buf
+      end
+      tfo.write(zipper.deflate(nil, Zlib::FINISH))
+      tfo.addTag("compressed") if tfo.respond_to?(:addTag)
+      tfo.close
+    end
+  end
+
+  def decompressFile(src, dest)
+    if @verbose
+      log_puts "\tDECOMPRESS: #{src}"
+      log_puts "\t      TO: #{dest}"
+    end
+    @fromFs.fileOpen(src) do |ffo|
+      tfo = @toFs.fileOpen(dest, "wb")
+      unzipper = Zlib::Inflate.new
+      while (buf = ffo.read(4096))
+        unzipper << buf
+      end
+      tfo.write(unzipper.inflate(nil))
+      tfo.close
+    end
+  end
+end # class MiqFsUpdate

data/lib/fs/MiqMountManager.rb

@@ -0,0 +1,209 @@
+require 'fs/MiqFS/MiqFS'
+require 'VolumeManager/MiqNativeVolumeManager'
+require 'fs/MountManagerProbe'
+
+class MiqMountManager < MiqFS
+  attr_accessor :rootVolume, :volumes, :guestOS, :fileSystems, :devHash, :osNames, :volMgr, :noFsVolumes, :payloads
+
+  def self.mountVolumes(volMgr, vmCfg, ost = nil)
+    rootTrees = []
+    noFsVolumes = []
+    volMgr.visibleVolumes.each do |dobj|
+      $log.debug("MiqMountManager.mountVolumes >> fileName=#{dobj.dInfo.fileName}, partition=#{dobj.partNum}") if $log
+      fs = MiqFS.getFS(dobj)
+      if fs.nil?
+        $log.debug "MiqMountManager.mountVolumes << SKIPPING because no file system" if $log
+        noFsVolumes << dobj
+        next
+      end
+      if (rsm = MountManagerProbe.getRootMod(fs))
+        rootTrees << new(rsm, dobj, volMgr, vmCfg, ost)
+      end
+    end
+    if volMgr.kind_of?(MiqNativeVolumeManager)
+      require 'MetakitFS'
+      rootTrees.each { |rt| rt.findPayload(noFsVolumes) } unless noFsVolumes.empty?
+    end
+    rootTrees
+  end # def self.mountVolumes
+
+  def initialize(rootModule, rootVolume, volMgr, vmCfg, _ost = nil)
+    @volMgr     = volMgr
+    @rootVolume   = rootVolume
+    @volumes    = volMgr.visibleVolumes
+    @noFsVolumes  = noFsVolumes
+    @vmConfig   = vmCfg
+    @fileSystems  = []
+    @allFileSystems = []
+    @payloads   = []
+    @devHash    = nil
+
+    super(rootModule, rootVolume)
+  end # def initialize
+
+  def findPayload(noFsVolumes)
+    $log.debug "MiqMountManager.findPayload: searching for payloads:" if $log
+    noFsVolumes.each do |v|
+      next unless v.respond_to?(:devFile)
+      if v.devFile
+        $log.debug "\tMiqMountManager.findPayload: devFile = #{v.devFile}" if $log
+        v.mkfile = v.devFile
+        unless MetakitFS.supported?(v)
+          $log.debug "\tMiqMountManager.findPayload: devFile = #{v.devFile} not mkfs, skipping" if $log
+          v.mkfile = nil
+          next
+        end
+        mkFs = MiqFS.new(MetakitFS, v)
+        if mkFs.fsId == "MIQPAYLOAD"
+          $log.debug "\tMiqMountManager.findPayload: payload found devFile = #{v.devFile}" if $log
+          @payloads << mkFs
+        else
+          $log.debug "\tMiqMountManager.findPayload: devFile = #{v.devFile} not payload, fsId = #{mkFs.fsId}" if $log
+        end
+      else
+        $log.debug "\tMiqMountManager.findPayload: devFile not set, fileName = #{v.dInfo.fileName}" if $log
+      end
+    end
+  end
+
+  #
+  # Override standard MiqFS methods to account for mount indirection.
+  # The getFsPath method is defined by the OS-specific mount modules.
+  #
+
+  def umount
+    @allFileSystems.each(&:umount)
+  end
+
+  # Return free space in file system.
+  def freeBytes
+    fs, _p = getFsPath(@cwd)
+    fs.freeBytes
+  end
+
+  #
+  # Directory instance methods
+  #
+
+  def dirEntries(*dir)
+    fs, p = getFsPath(dir)
+    fs.dirEntries(p)
+  end
+
+  def dirForeach(*dir, &block)
+    fs, p = getFsPath(dir)
+    fs.dirForeach(p, &block)
+  end
+
+  def dirGlob(glb, *flags, &block)
+    fs, p = getFsPath(@cwd)
+    fs.chdir(p)
+    return fs.dirGlob(glb, &block) if flags.length == 0
+    fs.dirGlob(glb, flags[0], &block)
+  end
+
+  def dirMkdir(dir)
+    fs, p = getFsPath(dir)
+    fs.dirMkdir(p)
+  end
+
+  def dirRmdir(dir)
+    fs, p = getFsPath(dir)
+    fs.dirRmdir(p)
+  end
+
+  #
+  # File instance methods
+  #
+
+  def fileExists?(f)
+    fs, p = getFsPath(f)
+    fs.fileExists?(p)
+  end
+
+  def fileFile?(f)
+    fs, p = getFsPath(f)
+    fs.fileFile?(p)
+  end
+
+  def fileDirectory?(f)
+    fs, p = getFsPath(f)
+    fs.fileDirectory?(p)
+  end
+
+  def fileSymLink?(f)
+    fs, p = getFsPath(f)
+    fs.fileSymLink?(p)
+  end
+
+  def fileOpen(f, mode = "r", &block)
+    fs, p = getFsPath(f)
+    fs.fileOpen(p, mode, &block)
+  end
+
+  def fileClose(mfobj)
+    @mfobj.fs.fileClose(mfobj)
+  end
+
+  def fileDelete(f)
+    fs, p = getFsPath(f)
+    fs.fileDelete(p)
+  end
+
+  def fileSize(f)
+    fs, p = getFsPath(f)
+    fs.fileSize(p)
+  end
+
+  def fileAtime(f)
+    fs, p = getFsPath(f)
+    fs.fileAtime(p)
+  end
+
+  def fileCtime(f)
+    fs, p = getFsPath(f)
+    fs.fileCtime(p)
+  end
+
+  def fileMtime(f)
+    fs, p = getFsPath(f)
+    fs.fileMtime(p)
+  end
+
+  def fileAtime_obj(fo)
+    fo.fs.fileAtime_obj(fo)
+  end
+
+  def fileCtime_obj(fo)
+    fo.fs.fileCtime_obj(fo)
+  end
+
+  def fileMtime_obj(fo)
+    fo.fs.fileMtime_obj(fo)
+  end
+
+  def internalPath(p)
+    fs, np = getFsPath(p)
+    fs.internalPath(np)
+  end
+
+  def toXml(doc = nil)
+    doc = MiqXml.createDoc(nil) unless doc
+
+    doc.add_element 'FileSystems'
+    @fileSystems.each do |fsd|
+      $miqOut.puts "FS: #{fsd.fsSpec}, Mounted on: #{fsd.mountPoint}, Type: #{fsd.fs.fsType}, Free bytes: #{fsd.fs.freeBytes}"
+    end
+    doc
+  end
+
+  private
+
+  def saveFs(fs, mp, fsSpec)
+    fsd = OpenStruct.new
+    fsd.fs = fs
+    fsd.mountPoint = mp
+    fsd.fsSpec = fsSpec
+    @fileSystems << fsd
+  end
+end # class MiqMountManager

data/lib/fs/MiqNativeMountManager.rb

@@ -0,0 +1,101 @@
+require 'log4r'
+require 'metadata/VmConfig/GetNativeCfg'
+require 'VolumeManager/MiqNativeVolumeManager'
+require 'fs/MiqMountManager'
+
+module MiqNativeMountManager
+  def self.mountVolumes
+    cfg = GetNativeCfg.new
+    volMgr = MiqNativeVolumeManager.new(cfg)
+
+    (MiqMountManager.mountVolumes(volMgr, cfg))
+  end
+end # module MiqNativeMountManager
+
+if __FILE__ == $0
+  #
+  # Formatter to output log messages to the console.
+  #
+  class ConsoleFormatter < Log4r::Formatter
+    def format(event)
+      (event.data.kind_of?(String) ? event.data : event.data.inspect) + "\n"
+    end
+  end
+  $log = Log4r::Logger.new 'toplog'
+  $log.level = Log4r::DEBUG
+  Log4r::StderrOutputter.new('err_console', :formatter => ConsoleFormatter)
+  $log.add 'err_console'
+
+  puts "Log debug?: #{$log.debug?}"
+
+  rootTrees = MiqNativeMountManager.mountVolumes
+
+  if rootTrees.nil? || rootTrees.empty?
+    puts "No root filesystems detected"
+    exit
+  end
+
+  $miqOut = $stdout
+  rootTrees.each do |r|
+    r.toXml(nil)
+  end
+
+  exit
+
+  rootTree = rootTrees[0]
+
+  ts = Time.now
+  fn = "Test-#{ts.year}#{ts.month}#{ts.day}-#{ts.hour}#{ts.min}#{ts.sec}"
+  dn = "Dir-#{fn}"
+  tstring = ts.to_s + "\n"
+
+  if rootTree.guestOS == "Linux"
+    puts
+    puts "Files in /:"
+    rootTree.dirForeach("/") { |f| puts "\t#{f}" }
+
+    # puts
+    # puts "All files in /test_mount:"
+    # rootTree.findEach("/test_mount") { |f| puts "\t#{f}" }
+    #
+    # tdn = File.join("/test_mount", dn)
+    # tfn = File.join(tdn, fn)
+  elsif rootTree.guestOS == "Windows"
+    puts
+    puts "Files in C:/"
+    rootTree.dirForeach("C:/") { |f| puts "\t#{f}" }
+
+    # ["E:/", "F:/"].each do |drive|
+    #   puts
+    #   puts "All files in #{drive}"
+    #   rootTree.findEach(drive) { |f| puts "\t#{f}" }
+    # end
+    #
+    # tdn = File.join("F:/", dn)
+    # tfn = File.join(tdn, fn + ".txt")
+  else
+    puts "Unknown guest OS: #{rootTree.guestOS}"
+  end
+
+  puts
+  puts "*** Payloads:"
+  rootTree.payloads.each { |p| puts "\t#{p.dobj.devFile}" }
+
+  # puts
+  # puts "*** Creating Directory: #{tdn}"
+  # rootTree.dirMkdir(tdn)
+  # puts "*** Creating file: #{tfn}"
+  # rootTree.copyIn(".", tdn, true)
+  # fo = rootTree.fileOpen(tfn, "w")
+  # fo.write(tstring)
+  # fo.close
+  # ip = rootTree.internalPath(tfn)
+  # puts "\tInternal path: #{ip}"
+  # puts "Contents:"
+  # `cat #{ip}`.each { |l| puts "\t#{l}" }
+  # puts "End contents"
+  # puts "*** Copied files:"
+  # rootTree.findEach(tdn) { |f| puts "\t#{f}" }
+
+  rootTrees.each(&:umount)
+end

data/lib/fs/MountManagerProbe.rb

@@ -0,0 +1,29 @@
+module MountManagerProbe
+  MODDIR = File.expand_path(File.join(File.dirname(__FILE__), "modules"))
+
+  PROBE_FILES = Dir.glob(File.join(MODDIR, "*Probe.rb*"))
+  PROBE_FILES.each do |p|
+    p.slice!(0..MODDIR.length)
+    p.chomp!(".enc")
+    p.chomp!(".rb")
+  end
+  # Reorder known probes for optimization
+  PROBE_FILES.unshift("WinMountProbe") if PROBE_FILES.delete("WinMountProbe")
+
+  def self.getRootMod(dobj, probes = nil)
+    probes = PROBE_FILES if probes.nil?
+    probes = [probes] unless probes.kind_of?(Array)
+
+    probes.each do |pmod|
+      $log.debug "MIQ(MountManagerProbe-getRootMod) Mount Manager probe attempting [#{pmod}]"
+      require_relative "modules/#{pmod}"
+      if Object.const_get(pmod).probe(dobj)
+        mod = pmod.chomp("Probe")
+        $log.info "MIQ(MountManagerProbe-getRootMod) Mount Manager probe detected [#{mod}]"
+        require_relative "modules/#{mod}"
+        return Object.const_get(mod)
+      end
+    end
+    nil
+  end
+end