manageiq-smartstate 0.1.0

data/lib/disk/modules/RawBlockIO.rb

@@ -0,0 +1,116 @@
+require 'linux_block_device'
+require 'memory_buffer'
+
+class RawBlockIO
+  MIN_SECTORS_TO_CACHE = 64
+
+  def initialize(filename, mode = "r")
+    # We must start with a block special file
+    raise "RawBlockIO: #{filename} is not a blockSpecial file" unless File.stat(filename).ftype == 'blockSpecial'
+
+    @rawDisk_file = File.open(filename, mode)
+
+    # Enable directio (raw) if supported
+    if defined? File::DIRECT
+      require 'fcntl'
+      @rawDisk_file.fcntl(Fcntl::F_SETFL, File::DIRECT)
+    end
+
+    @blockSize      = 512
+    @filename       = filename
+    @mode           = mode
+    @size           = LinuxBlockDevice.size(@rawDisk_file.fileno)
+    @sizeInBlocks   = @size / @blockSize
+    @startByteAddr  = 0
+    @endByteAddr    = @size - 1
+    @lbaEnd         = @sizeInBlocks - 1
+    @seekPos        = 0
+
+    $log.debug "RawBlockIO: opened #{@filename}, size = #{@size} (#{@sizeInBlocks} blocks)"
+  end
+
+  def read(len)
+    return nil if @seekPos >= @endByteAddr
+    len = @endByteAddr - @seekPos if (@seekPos + len) > @endByteAddr
+
+    startSector, startOffset = @seekPos.divmod(@blockSize)
+    endSector = (@seekPos + len - 1) / @blockSize
+    numSector = endSector - startSector + 1
+
+    rBuf = breadCached(startSector, numSector)
+    @seekPos += len
+
+    rBuf[startOffset, len]
+  end
+
+  def write(buf, len)
+    return nil if @seekPos >= @endByteAddr
+    len = @endByteAddr - @seekPos if (@seekPos + len) > @endByteAddr
+
+    startSector, startOffset = @seekPos.divmod(@blockSize)
+    endSector = (@seekPos + len - 1) / @blockSize
+    numSector = endSector - startSector + 1
+
+    rBuf = bread(startSector, numSector)
+    rBuf[startOffset, len] = buf[0, len]
+
+    bwrite(startSector, numSector, rBuf)
+    @seekPos += len
+
+    len
+  end
+
+  def seek(amt, whence = IO::SEEK_SET)
+    case whence
+    when IO::SEEK_CUR
+      @seekPos += amt
+    when IO::SEEK_END
+      @seekPos = @endByteAddr + amt
+    when IO::SEEK_SET
+      @seekPos = amt + @startByteAddr
+    end
+    @seekPos
+  end
+
+  attr_reader :size
+
+  def close
+    @rawDisk_file.close
+  end
+
+  def bread(startSector, numSectors)
+    # $log.debug "RawBlockIO.bread: startSector = #{startSector}, numSectors = #{numSectors}, @lbaEnd = #{@lbaEnd}"
+    return nil if startSector > @lbaEnd
+    numSectors = @sizeInBlocks - startSector if (startSector + numSectors) > @sizeInBlocks
+
+    @rawDisk_file.sysseek(startSector * @blockSize, IO::SEEK_SET)
+    @rawDisk_file.sysread(numSectors * @blockSize, @cache)
+  end
+
+  def bwrite(startSector, numSectors, buf)
+    return nil if startSector > @lbaEnd
+    numSectors = @sizeInBlocks - startSector if (startSector + numSectors) > @sizeInBlocks
+
+    @rawDisk_file.sysseek(startSector * @blockSize, IO::SEEK_SET)
+    @rawDisk_file.syswrite(buf, numSectors * @blockSize)
+  end
+
+  def breadCached(startSector, numSectors)
+    # $log.debug "RawBlockIO.breadCached: startSector = #{startSector}, numSectors = #{numSectors}"
+    if @cacheRange.nil? || !@cacheRange.include?(startSector) || !@cacheRange.include?(startSector + numSectors - 1)
+      sectorsToRead = [MIN_SECTORS_TO_CACHE, numSectors].max
+      @cache        = MemoryBuffer.create_aligned(@blockSize, @blockSize * sectorsToRead)
+      bread(startSector, sectorsToRead)
+      sectorsRead   = @cache.length / @blockSize
+      endSector     = startSector + sectorsRead - 1
+      @cacheRange   = Range.new(startSector, endSector)
+    end
+
+    sectorOffset = startSector - @cacheRange.first
+    bufferOffset = sectorOffset * @blockSize
+    length       = numSectors * @blockSize
+    # $log.debug "RawBlockIO.breadCached: bufferOffset = #{bufferOffset}, length = #{length}"
+
+    @cache[bufferOffset, length]
+  end
+end

data/lib/disk/modules/RawDisk.rb

@@ -0,0 +1,45 @@
+require 'disk/modules/MiqLargeFile'
+
+module RawDisk
+  def d_init
+    self.diskType = "Raw"
+    self.blockSize = 512
+
+    if dInfo.mountMode.nil? || dInfo.mountMode == "r"
+      dInfo.mountMode = "r"
+      fileMode = "r"
+    elsif dInfo.mountMode == "rw"
+      fileMode = "r+"
+    else
+      raise "Unrecognized mountMode: #{dInfo.mountMode}"
+    end
+
+    @dOffset = dInfo.offset
+    @rawDisk_file = MiqLargeFile.open(dInfo.fileName, fileMode)
+  end
+
+  def getBase
+    self
+  end
+
+  def d_read(pos, len, offset = 0)
+    pos += @dOffset if @dOffset
+    @rawDisk_file.seek(pos - offset, IO::SEEK_SET)
+    @rawDisk_file.read(len)
+  end
+
+  def d_write(pos, buf, len, offset = 0)
+    pos += @dOffset if @dOffset
+    @rawDisk_file.seek(pos - offset, IO::SEEK_SET)
+    @rawDisk_file.write(buf, len)
+  end
+
+  def d_close
+    @rawDisk_file.close
+  end
+
+  # Disk size in sectors.
+  def d_size
+    @rawDisk_file.size / @blockSize
+  end
+end

data/lib/disk/modules/RawDiskProbe.rb

@@ -0,0 +1,7 @@
+module RawDiskProbe
+  def self.probe(ostruct)
+    return nil unless ostruct.fileName
+    return("RawDisk") if ostruct.rawDisk || File.extname(ostruct.fileName).downcase == ".img"
+    (nil)
+  end
+end

data/lib/disk/modules/RhevmDescriptor.rb

@@ -0,0 +1,167 @@
+require 'pathname'
+
+module RhevmDescriptor
+  def d_init
+    # Make sure this is a descriptor.
+    desc, defs = parseDescriptor(dInfo.Descriptor)
+
+    # Make sure each disk is there.
+    defs.each do|diskDef|
+      filename = buildFilename(diskDef[:filename])
+      raise "No disk file: #{filename}" unless File.exist?(filename)
+    end
+
+    # Init needed stff.
+    self.diskType = "Rhevm Descriptor"
+    self.blockSize = 512
+    @desc     = desc  # This disk descriptor.
+    @defs     = defs  # Disk extent definitions.
+    @ostructs = []    # Component OpenStructs for disk objects.
+    @disks    = []    # Component MiqDisk objects (one for each disk).
+
+    # If there's a parent parse it first (all subordinate disks need a ref to parent).
+    if desc.key?(:puuid) && desc[:puuid] != '00000000-0000-0000-0000-000000000000'
+      @parentOstruct = OpenStruct.new
+
+      # Get the parentFileNameHint and be sure it is relative to the descriptor file's directory
+      parentFileName = buildFilename(desc[:puuid])
+      # puts "#{self.class.name}: Getting parent disk file [#{parentFileName}]"
+
+      @parentOstruct.fileName = parentFileName
+      @parentOstruct.mountMode = dInfo.mountMode
+      @parentOstruct.hardwareId = dInfo.hardwareId if dInfo.baseOnly
+      d = MiqDisk.getDisk(@parentOstruct)
+      raise "MiqDisk#getDisk returned nil for parent disk #{@parentOstruct.fileName}" if d.nil?
+      @parent = d
+      return if dInfo.baseOnly
+    end
+
+    # Instance MiqDisks for each disk definition.
+    defs.each do |diskDef|
+      thisO = OpenStruct.new
+      thisO.Descriptor = dInfo.Descriptor if dInfo.Descriptor
+      thisO.parent = @parent
+      thisO.fileName = buildFilename(diskDef[:filename])
+      thisO.offset = diskDef[:offset]
+      thisO.rawDisk = true if diskDef[:format].to_s.include?('RAW')
+      thisO.mountMode = dInfo.mountMode
+      @ostructs << thisO
+      d = MiqDisk.getDisk(thisO)
+      raise "MiqDisk#getDisk returned nil for component disk #{thisO.fileName}" if d.nil?
+      @disks << d
+    end
+  end
+
+  def getBase
+    @parent || self
+  end
+
+  def d_read(pos, len)
+    # $log.debug "RhevmDescriptor.d_read << pos #{pos} len #{len}" if $log && $log.debug?
+    # Get start and end extents.
+    dStart = getTargetDiskIndex((pos / @blockSize).ceil)
+    dEnd   = getTargetDiskIndex(((pos + len) / @blockSize).ceil)
+
+    if dStart == dEnd
+      # Case: single extent.
+      retBytes = @disks[dStart].d_read(pos, len, getDiskByteOffset(dStart))
+    else
+      # Case: span extents.
+      retBytes = ""; bytesRead = 0
+      dStart.upto(dEnd) do |diskIdx|
+        readLen = @disks[diskIdx].d_size
+
+        # Adjust length for start and end extents.
+        readLen -= pos if diskIdx == dStart
+        readLen -= (len - bytesRead) if diskIdx == dEnd
+
+        # Read.
+        retBytes << @disks[diskIdx].d_read(pos + bytesRead, readLen, getDiskByteOffset(diskIdx))
+        bytesRead += readLen
+      end
+    end
+    # $log.debug "RhevmDescriptor.d_read >> retBytes.length #{retBytes.length}" if $log && $log.debug?
+    retBytes
+  end
+
+  def d_write(pos, buf, len)
+    dStart = getTargetDiskIndex((pos / @blockSize).ceil)
+    dEnd   = getTargetDiskIndex(((pos + len) / @blockSize).ceil)
+
+    # Case: single extent.
+    return @disks[dStart].d_write(pos, buf, len, getDiskByteOffset(dStart)) if dStart == dEnd
+
+    # Case: span extents.
+    bytesWritten = 0
+    dStart.upto(dEnd) do |diskIdx|
+      writeLen = @disks[diskIdx].d_size
+
+      # Adjust length for start and end extents.
+      writeLen -= pos if diskIdx == dStart
+      writeLen -= (len - bytesWritten) if diskIdx == dEnd
+
+      # Write.
+      bytesWritten += @disks[diskIdx].d_write(pos + bytesWritten, writeLen, getDiskByteOffset(diskIdx))
+    end
+    bytesWritten
+  end
+
+  # Close all disks.
+  def d_close
+    @parent.close unless @parent.nil?
+    @disks.each(&:close)
+  end
+
+  # Return size in sectors.
+  def d_size
+    @desc[:size].to_i
+  end
+
+  # Get target disk index based on sector number.
+  def getTargetDiskIndex(sector)
+    dIdx = -1; total = 0
+    0.upto(@defs.size - 1) do|idx|
+      total += @defs[idx][:size].to_i
+      if total >= sector
+        dIdx = idx
+        break
+      end
+    end
+    raise "Sector is past end of disk: #{sector}" if dIdx == -1
+    raise "Disk object is nil for #{sector}" if @disks[dIdx].nil?
+    dIdx
+  end
+
+  # Get beginning byte offset of target disk.
+  def getDiskByteOffset(target)
+    total = 0
+    0.upto(@defs.size - 1) do|idx|
+      break if idx == target
+      total += @defs[idx]['size'].to_i
+    end
+    total * @blockSize
+  end
+
+  def parseDescriptor(descriptor)
+    desc = {}
+    descriptor.each_line do |line|
+      line.strip!
+      break if line == 'EOF'
+      next unless line.include?('=')
+      key, *value = line.split('=')
+      desc[key.downcase.to_sym] = value = value.join('=')
+    end
+    desc[:offset] = 0
+    desc[:filename] = File.basename(dInfo.fileName)
+    return desc, [desc]
+  end
+
+  def buildFilename(parent_uuid)
+    parentFileName = Pathname.new(parent_uuid)
+    if parentFileName.absolute?
+      parentFileName = parentFileName.relative_path_from(Pathname.new(dInfo.fileName).dirname)
+      $log.debug "#{self.class.name}: Parent disk file is absolute. Using relative path [#{parentFileName}]" if $log
+    end
+    File.join(File.dirname(dInfo.fileName), parentFileName.to_s.tr("\\", "/"))
+  end
+end # module RhevmDescriptor

data/lib/disk/modules/RhevmDiskProbe.rb

@@ -0,0 +1,52 @@
+module RhevmDiskProbe
+  DESC_MOD = "RhevmDescriptor"
+  QCOW_MOD = "QcowDisk"
+  RAW_MOD  = 'RawDisk'
+
+  def self.probe(ostruct)
+    return nil unless ostruct.fileName
+
+    ext = File.extname(ostruct.fileName).downcase
+    return nil unless ext.length.zero?
+
+    format = ostruct.format.to_s.downcase
+    return RAW_MOD  if format == 'raw'
+    return QCOW_MOD if format == 'cow'
+
+    descriptor_file = ostruct.fileName + '.meta'
+    return nil unless File.exist?(descriptor_file)
+
+    # If this ostruct already has a descriptor don't bother checking.
+    # NOTE: If it does have a descriptor, we're coming from RhevmDescriptor.rb
+    #       trying to open a disk - so don't regress infinitely.
+    if ostruct.Descriptor.nil?
+      # Get descriptor metadata
+      f = File.open(descriptor_file, "r")
+      descriptor = f.read; f.close
+      if descriptor.include?('EOF')
+        ostruct.Descriptor = descriptor
+        desc = parseDescriptor(descriptor)
+        return RAW_MOD if desc[:format].to_s.include?('RAW')
+        return QCOW_MOD if desc[:format].to_s.include?('COW')
+        return DESC_MOD
+      end
+    else
+      desc = parseDescriptor(ostruct.Descriptor)
+      return RAW_MOD  if desc[:format].to_s.include?('RAW')
+      return QCOW_MOD if desc[:format].to_s.include?('COW')
+    end
+    nil
+  end
+
+  def self.parseDescriptor(descriptor)
+    desc = {}
+    descriptor.each_line do |line|
+      line.strip!
+      break if line == 'EOF'
+      next unless line.include?('=')
+      key, *value = line.split('=')
+      desc[key.downcase.to_sym] = value = value.join('=')
+    end
+    desc
+  end
+end # module RhevmDiskProbe

data/lib/disk/modules/VMWareCowdDisk.rb

@@ -0,0 +1,207 @@
+require 'disk/modules/MiqLargeFile'
+require 'binary_struct'
+require 'memory_buffer'
+
+module VMWareCowdDisk
+  COWD_EXTENT_HEADER = BinaryStruct.new([
+    'L', 'magicNumber',
+    'L', 'version',
+    'L', 'flags',
+    'L', 'numSectors',
+    'L', 'grainSize',
+    'L', 'gdOffset',
+    'L', 'numGDEntries',
+    'L', 'freeSector',
+  ])
+
+  SIZEOF_COWD_EXTENT_HEADER = COWD_EXTENT_HEADER.size
+  GTE_SIZE = 4
+  GDE_SIZE = 4
+  GDE_COVERAGE = 2097152
+  ENTRIES_PER_TABLE = 4096
+  GT_SECTORS = 32
+
+  def d_init
+    self.diskType = "VMWare CopyOnWrite"
+    self.blockSize = 512
+    if dInfo.mountMode.nil? || dInfo.mountMode == "r"
+      dInfo.mountMode = "r"
+      fileMode = "r"
+    elsif dInfo.mountMode == "rw"
+      fileMode = "r+"
+    else
+      raise "Unrecognized mountMode: #{dInfo.mountMode}"
+    end
+    @dParent = @dInfo.parent
+
+    @vmwareCowDisk_file = MiqLargeFile.open(dInfo.fileName, fileMode)
+    buf = @vmwareCowDisk_file.read(SIZEOF_COWD_EXTENT_HEADER)
+    @EsxSparseHeader = OpenStruct.new(COWD_EXTENT_HEADER.decode(buf))
+
+    @grainSize = @EsxSparseHeader.grainSize
+    @grainBytes = @grainSize * blockSize
+    @capacity = @EsxSparseHeader.numSectors * blockSize
+    @gdStart = @EsxSparseHeader.gdOffset * blockSize
+    @gdSize = @EsxSparseHeader.numGDEntries * GDE_SIZE
+  end
+
+  def getBase
+    self
+  end
+
+  def d_read(pos, len, _offset = 0)
+    gnStart, goStart = grainPos(pos)
+    gnEnd, goEnd = grainPos(pos + len)
+
+    if gnStart == gnEnd
+      grainPos = getGTE(gnStart)
+      if grainPos == 0
+        return @dParent ? @dParent.d_read(pos, len) : MemoryBuffer.create(len)
+      else
+        @vmwareCowDisk_file.seek(grainPos + goStart, IO::SEEK_SET)
+        return @vmwareCowDisk_file.read(len)
+      end
+    end
+
+    bytesRead = 0
+    rv = ''
+    gnStart.upto(gnEnd) do |gn|
+      so = 0
+      l = @grainBytes
+
+      if gn == gnStart
+        so = goStart
+        l -= so
+      end
+      l -= (@grainBytes - goEnd) if gn == gnEnd
+
+      gp = getGTE(gn)
+      # Mods for parent link.
+      if gp == 0
+        rv << (@dParent ? @dParent.d_read(pos + bytesRead, l) : MemoryBuffer.create(l))
+      else
+        @vmwareCowDisk_file.seek(gp + so, IO::SEEK_SET)
+        rv << @vmwareCowDisk_file.read(l)
+      end
+      bytesRead += l
+    end
+    rv
+  end
+
+  def d_write(pos, buf, len, _offset = 0)
+    gnStart, goStart = grainPos(pos)
+    gnEnd, goEnd = grainPos(pos + len)
+
+    if gnStart == gnEnd
+      grainPos = getGTE(gnStart)
+      grainPos = allocGrain(gnStart) if grainPos == 0
+      @vmwareCowDisk_file.seek(grainPos + goStart, IO::SEEK_SET)
+      return @vmwareCowDisk_file.write(buf, len)
+    end
+
+    bytesWritten = 0
+    gnStart.upto(gnEnd) do |gn|
+      so = 0
+      l = @grainBytes
+
+      if gn == gnStart
+        so = goStart
+        l -= so
+      end
+      l -= (@grainBytes - goEnd) if gn == gnEnd
+
+      gp = getGTE(gn)
+      gp = allocGrain(gn) if gp == 0
+      raise "Disk is full" if gp == -1
+      @vmwareCowDisk_file.seek(gp + so, IO::SEEK_SET)
+      bytesWritten += @vmwareCowDisk_file.write(buf[bytesWritten, l], l)
+    end
+    bytesWritten
+  end
+
+  def d_close
+    @vmwareCowDisk_file.close
+  end
+
+  # Disk size in sectors.
+  def d_size
+    @capacity / @blockSize
+  end
+
+  private
+
+  def grainPos(pos)
+    gn = pos / @grainBytes
+    go = pos - (gn * @grainBytes)
+    return gn, go
+  end
+
+  def getGTE(gn)
+    return 0 if seekGTE(gn) == -1
+    gte = @vmwareCowDisk_file.read(GTE_SIZE).unpack('L')[0]
+    gte * blockSize
+  end
+
+  def putGTE(gn, sector)
+    seekGTE(gn)
+    @vmwareCowDisk_file.write([sector].pack('L'), GTE_SIZE)
+  end
+
+  def allocGrain(gn)
+    allocGT(gn) if getGDE(gn) == 0
+    freeSector = @EsxSparseHeader.freeSector
+    raise "Disk full." if freeSector + @grainSize > @EsxSparseHeader.numSectors
+    thisGrain = freeSector * blockSize
+    putGTE(gn, freeSector)
+    freeSector += @grainSize
+    updateFreeSector(freeSector)
+    copyOnWrite(gn) if @dParent
+    thisGrain
+  end
+
+  def seekGTE(gn)
+    seekGDE(gn)
+    gde = @vmwareCowDisk_file.read(GDE_SIZE).unpack('L')[0] * blockSize
+    return -1 if gde == 0
+    @vmwareCowDisk_file.seek(gde + gn.modulo(ENTRIES_PER_TABLE) * GTE_SIZE, IO::SEEK_SET)
+  end
+
+  def getGDE(gn)
+    seekGDE(gn)
+    @vmwareCowDisk_file.read(GDE_SIZE).unpack('L')[0]
+  end
+
+  def putGDE(gn, sector)
+    seekGDE(gn)
+    @vmwareCowDisk_file.write([sector].pack('L'), GDE_SIZE)
+  end
+
+  def seekGDE(gn)
+    grainDirEntOfs = (gn * @grainBytes).divmod(GDE_COVERAGE)[0]
+    @vmwareCowDisk_file.seek(@gdStart + grainDirEntOfs * GDE_SIZE, IO::SEEK_SET)
+  end
+
+  def allocGT(gn)
+    freeSector = @EsxSparseHeader.freeSector
+    raise "Disk full." if freeSector + GT_SECTORS > @EsxSparseHeader.numSectors
+    putGDE(gn, freeSector)
+    freeSector += GT_SECTORS
+    gtSize = ENTRIES_PER_TABLE * GTE_SIZE
+    gt = MemoryBuffer.create(gtSize)
+    @vmwareCowDisk_file.seek(freeSector * blockSize, IO::SEEK_SET)
+    @vmwareCowDisk_file.write(gt, gtSize)
+    updateFreeSector(freeSector)
+  end
+
+  def updateFreeSector(freeSector)
+    @EsxSparseHeader.freeSector = freeSector
+    @vmwareCowDisk_file.seek(SIZEOF_COWD_EXTENT_HEADER - 4, IO::SEEK_SET)
+    @vmwareCowDisk_file.write([freeSector].pack('L'), 4)
+  end
+
+  def copyOnWrite(gn)
+    buf = @dParent.d_read(gn * blockSize, @grainBytes)
+    @vmwareCowDisk_file.seek(gn * blockSize, IO::SEEK_SET)
+    @vmwareCowDisk_file.write(buf, @grainBytes)
+  end
+end