manageiq-smartstate 0.1.0

data/lib/fs/MetakitFS/MetakitFS.rb

@@ -0,0 +1,530 @@
+# encoding: US-ASCII
+
+require 'mk4rb'
+
+module MetakitFS
+  MK_FENTRY   = "fentry[fpath:S,ftype:I,fsize:I,ftags:B,fdata:B]"
+  MK_HASHVW   = "sec[_H:I,_R:I]"
+
+  TYPE_FILE       = 1
+  TYPE_DIR        = 2
+  TYPE_SYMLINK    = 3
+
+  def self.supported?(dobj)
+    return(false) unless dobj.mkfile
+    storage = Metakit::Storage.open(dobj.mkfile, 1)
+    return(false) if storage.description != "#{MK_FENTRY},#{MK_HASHVW}"
+    (true)
+  end
+
+  def fs_init
+    raise "No metakit file has been specified" unless @dobj.mkfile
+    @storage = Metakit::Storage.open(@dobj.mkfile, 1)
+
+    newFs = false
+    if @storage.description != "#{MK_FENTRY},#{MK_HASHVW}"
+      raise "#{@dobj.mkfile} is not a MetakitFS" unless @dobj.create
+      newFs = true
+    end
+
+    self.fsType = "MetakitFS"
+
+    vData       = @storage.get_as MK_FENTRY
+    vSec        = @storage.get_as MK_HASHVW
+    @vFentry    = vData.hash(vSec, 1)
+
+    @pPath = Metakit::StringProp.new "fpath"
+    @pType = Metakit::IntProp.new    "ftype"
+    @pSize = Metakit::IntProp.new    "fsize"
+    @pTags = Metakit::BytesProp.new  "ftags"
+    @pData = Metakit::BytesProp.new  "fdata"
+
+    @findRow = Metakit::Row.new
+
+    #
+    # If we're creating a new file system, create the root directory.
+    #
+    if newFs
+      create("/", TYPE_DIR) if newFs
+      @storage.commit
+    end
+
+    labels = fs_tagValues("/", "LABEL")
+    @fsId = labels[0] unless labels.empty?
+  end
+
+  def fs_dirEntries(p)
+    raise "Directory: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    raise "#{p} is not a directory" if @pType.get(@vFentry[i]) != TYPE_DIR
+
+    data = @pData.get(@vFentry[i])
+    return [] if data.size == 0
+    data.contents.split("\0")
+  end
+
+  def fs_dirMkdir(p)
+    #
+    # If the target exists, succeed if it is a directory, otherwise fail.
+    #
+    if (i = getFileIndex(p)) >= 0
+      return if @pType.get(@vFentry[i]) == TYPE_DIR
+      raise "Cannot create directory, #{p}: file exists"
+    end
+
+    #
+    # Create the directory.
+    #
+    create(p, TYPE_DIR)
+
+    #
+    # Apply changes to database.
+    #
+    @storage.commit
+  end
+
+  def fs_dirRmdir(p)
+    #
+    # Before we can remove a directory, it must:
+    # exist, be a directory and be empty.
+    #
+    raise "#{p}: no such directory" if (di = getFileIndex(p)) < 0
+    dirRow = @vFentry[di]
+    raise "#{p}: not a directory" if @pType.get(dirRow) != TYPE_DIR
+    raise "#{p}: directory not empty" if @pSize.get(dirRow) != 0
+
+    rmCommon(p, di)
+  end
+
+  def fs_fileExists?(p)
+    return false if getFileIndex(p) < 0
+    true
+  end
+
+  def fs_fileFile?(p)
+    return false if (i = getFileIndex(p)) < 0
+    return true  if @pType.get(@vFentry[i]) == TYPE_FILE
+    false
+  end
+
+  def fs_fileDirectory?(p)
+    return false if (i = getFileIndex(p)) < 0
+    return true  if @pType.get(@vFentry[i]) == TYPE_DIR
+    false
+  end
+
+  def fs_fileSize(p)
+    raise "File: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    (@pSize.get(@vFentry[i]))
+  end
+
+  def fs_fileSize_obj(fobj)
+    (@pSize.get(fobj.fileRow))
+  end
+
+  def fs_fileAtime(p)
+    File.atime(p)
+  end
+
+  def fs_fileCtime(p)
+    File.ctime(p)
+  end
+
+  def fs_fileMtime(p)
+    File.mtime(p)
+  end
+
+  def fs_fileAtime_obj(fobj)
+    fobj.atime
+  end
+
+  def fs_fileCtime_obj(fobj)
+    fobj.ctime
+  end
+
+  def fs_fileMtime_obj(fobj)
+    fobj.mtime
+  end
+
+  def fs_fileOpen(p, mode = "r")
+    fread = fwrite = fcreate = ftruncate = fappend = false
+    mode.delete!("b")
+
+    case mode[0, 1]
+    when "r"
+      fread     = true
+      fwrite    = true if mode[-1, 1] == "+"
+    when "w"
+      fwrite    = true
+      fcreate   = true
+      ftruncate = true
+      fread     = true if mode[-1, 1] == "+"
+    when "a"
+      fwrite    = true
+      fcreate   = true
+      fappend   = true
+      fread     = true if mode[-1, 1] == "+"
+    else
+      raise "Unrecognized open mode: #{mode}"
+    end
+
+    fileRow = nil
+    fi = getFileIndex(p)
+
+    if fi < 0
+      #
+      # Should we create the file? If not, fail.
+      #
+      raise "#{p}: No such file" unless fcreate
+
+      #
+      # Create the file if it doesn't exist.
+      #
+      fileRow = create(p, TYPE_FILE)
+      fi = getFileIndex(p)
+    else
+      fileRow = @vFentry[fi]
+    end
+
+    fpos = 0
+    fsize = @pSize.get(fileRow)
+    if ftruncate && fsize != 0
+      @pSize.set fileRow, 0
+      @pData.set fileRow, Metakit::Bytes.new("", 0)
+      @storage.commit
+    elsif fappend
+      fpos = fsize
+    end
+
+    (MkFile.new(p, fileRow, fpos, fread, fwrite))
+  end # def fs_fileOpen
+
+  def fs_fileSeek(fobj, _offset, whence)
+    seekPos = 0
+    case whence
+    when IO::SEEK_CUR
+      seekPos = filePos + amt
+    when IO::SEEK_END
+      seekPos = fs_fileSize_obj(fobj) + amt
+    when IO::SEEK_SET
+      seekPos = amt
+    else
+      raise "Invalid whence value: #{whence}"
+    end
+    raise "Invalid seek position: #{seekPos}" if seekPos < 0 || seekPos > fs_fileSize_obj(fobj)
+
+    fobj.filePos = seekPos
+  end
+
+  def fs_fileRead(fobj, len)
+    dataRef = @pData.ref(fobj.fileRow)
+    rb = dataRef.access(fobj.filePos, len)
+    fobj.filePos += rb.size
+    return(rb.contents) if rb.contents.length > 0
+    (nil)
+  end
+
+  def fs_fileWrite(fobj, buf, len)
+    raise "fs_fileWrite: write length is larger than buffer" if len > buf.length
+    size = fs_fileSize_obj(fobj)
+
+    if fobj.filePos == size
+      #
+      # We're appending to the end of the file, so we can just use the
+      # modify operation to add the data to the end of the file.
+      #
+      dataRef = @pData.ref(fobj.fileRow)
+      data = Metakit::Bytes.new(buf, len)
+      dataRef.modify(data, size, len)
+      @pSize.set(fobj.fileRow, size + len)
+      fobj.filePos += len
+      @storage.commit
+      return(len)
+    end
+    #
+    # The Metakit modify operation inserts data. So if we need to overwrite
+    # existing data, we must read the whole file, modify the data, and write
+    # it out again.
+    #
+    dataStr = @pData.get(fobj.fileRow).contents
+    dataStr[fobj.filePos, len] = buf[0, len]
+    data = Metakit::Bytes.new(dataStr, dataStr.length)
+    @pData.set(fobj.fileRow, data)
+    @pSize.set(fobj.fileRow, dataStr.length)
+    @storage.commit
+    (len)
+  end
+
+  def fs_fileClose(_fobj)
+    nil
+  end
+
+  def fs_fileDelete(p)
+    #
+    # Before we can remove a directory, it must:
+    # exist, be a directory and be empty.
+    #
+    raise "#{p}: no such file" if (fi = getFileIndex(p)) < 0
+    dirRow = @vFentry[di]
+    raise "#{p}: is a directory" if @pType.get(dirRow) == TYPE_DIR
+
+    rmCommon(p, fi)
+  end
+
+  def tagAdd(p, tag)
+    fs_tagAdd(normalizePath(p), tag)
+  end
+
+  def fs_tagAdd(p, tag)
+    raise "Path: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    fileRow = @vFentry[i]
+    fs_tagAddCommon(fileRow, tag)
+  end
+
+  def fs_fileTagAdd(fobj, tag)
+    fs_tagAddCommon(fobj.fileRow, tag)
+  end
+
+  def fs_tagAddCommon(fileRow, tag)
+    tagStr = @pTags.get(fileRow).contents
+    tags = tagStr.split("\0")
+    return if tags.include? tag
+    tags << tag
+    tagStr = tags.join("\0")
+    @pTags.set(fileRow, Metakit::Bytes.new(tagStr, tagStr.length))
+    @storage.commit
+  end
+
+  def tagDelete(p, tag)
+    fs_tagDelete(normalizePath(p), tag)
+  end
+
+  def fs_tagDelete(p, tag)
+    raise "Path: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    fileRow = @vFentry[i]
+    fs_tagDeleteCommon(fileRow, tag)
+  end
+
+  def fs_fileTagDelete(fobj, tag)
+    fs_tagDeleteCommon(fobj.fileRow, tag)
+  end
+
+  def fs_tagDeleteCommon(fileRow, tag)
+    tagStr = @pTags.get(fileRow).contents
+    tags = tagStr.split("\0")
+    tags.delete(tag) { raise "Tag #{tag} not found" }
+    tagStr = tags.join("\0")
+    @pTags.set(fileRow, Metakit::Bytes.new(tagStr, tagStr.length))
+    @storage.commit
+  end
+
+  def tags(p)
+    fs_tags(normalizePath(p))
+  end
+
+  def fs_tags(p)
+    raise "Path: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    fileRow = @vFentry[i]
+    fs_tagsCommon(fileRow)
+  end
+
+  def fs_fileTags(fobj)
+    fs_tagsCommon(fobj.fileRow)
+  end
+
+  def fs_tagsCommon(fileRow)
+    tags = @pTags.get(fileRow)
+    return [] if tags.size == 0
+    tags.contents.split("\0")
+  end
+
+  def hasTagName?(p, tag)
+    fs_hasTagName?(normalizePath(p), tag)
+  end
+
+  def fs_hasTagName?(p, tagName)
+    raise "Path: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    fileRow = @vFentry[i]
+    fs_hasTagNameCommon?(fileRow, tagName)
+  end
+
+  def fs_fileHasTagName?(fobj, tagName)
+    fs_hasTagNameCommon?(fobj.fileRow, tagName)
+  end
+
+  def fs_hasTagNameCommon?(fileRow, tagName)
+    fs_tagsCommon(fileRow).each { |t| return true if t =~ /#{tagName}(=.*)*$/ }
+    false
+  end
+
+  def hasTag?(p, tag)
+    fs_hasTag?(normalizePath(p), tag)
+  end
+
+  def fs_hasTag?(p, tag)
+    raise "Path: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    fileRow = @vFentry[i]
+    fs_hasTagCommon?(fileRow, tag)
+  end
+
+  def fs_fileHasTag?(fobj, tag)
+    fs_tagsCommon(fobj.fileRow).include?(tag)
+  end
+
+  def fs_hasTagCommon?(fileRow, tag)
+    fs_tagsCommon(fileRow).include?(tag)
+  end
+
+  def tagValues(p, tag)
+    fs_tagValues(normalizePath(p), tag)
+  end
+
+  def fs_tagValues(p, tag)
+    raise "Path: #{p} does not exist" if (i = getFileIndex(p)) < 0
+    fileRow = @vFentry[i]
+    fs_tagValuesCommon(fileRow, tag)
+  end
+
+  def fs_fileTagValues(fobj, tag)
+    fs_tagValuesCommon(fobj.fileRow, tag)
+  end
+
+  def fs_tagValuesCommon(fileRow, tag)
+    values = []
+    tagFound = false
+    fs_tagsCommon(fileRow).each do |t|
+      if t =~ /#{tag}(=(.*))*$/
+        tagFound = true
+        values << $2 if $2
+      end
+    end
+
+    return [] unless tagFound
+    values
+  end
+
+  def fs_fileObjExtend(fo)
+    fo.extend MkFileMod
+  end
+
+  private
+
+  def getFileIndex(p)
+    @pPath.set @findRow, p
+    (@vFentry.find(@findRow, 0))
+  end
+
+  def rmCommon(p, idx)
+    #
+    # If we get here, the parent hast to exist and it has to be a directory,
+    # so if it isn't, there's a bug somewhere.
+    #
+    parent = File.dirname(p)
+    raise "[BUG] #{parent}: no such directory" if (pi = getFileIndex(parent)) < 0
+    parentRow = @vFentry[pi]
+    raise "[BUG] #{parent}: not a directory" if @pType.get(parentRow) != TYPE_DIR
+
+    #
+    # First, remove the reference in the parent directory.
+    #
+    dirName = File.basename(p)
+    dataStr = @pData.get(parentRow).contents
+    dirEnts = dataStr.split("\0")
+    dirEnts.delete(dirName) { raise "[BUG] Directory #{parent} does not contain entry for #{dirName}" }
+    dataStr = dirEnts.join("\0")
+    @pData.set(parentRow, Metakit::Bytes.new(dataStr, dataStr.length))
+
+    #
+    # Then, remove the directory's row from the database.
+    #
+    @vFentry.remove_at(idx)
+
+    #
+    # Apply changes to database.
+    #
+    @storage.commit
+  end
+
+  def create(p, type)
+    #
+    # Fail if the parent component of the path doesn't exist,
+    # or if it is not a directory.
+    #
+    if p != "/"
+      parent = File.dirname(p)
+      raise "#{parent}: no such directory" if (i = getFileIndex(parent)) < 0
+      parentRow = @vFentry[i]
+      raise "#{parent}: not a directory" if @pType.get(parentRow) != TYPE_DIR
+    end
+
+    #
+    # Create the new empty file or directory.
+    #
+    newRow = Metakit::Row.new
+    @pPath.set newRow, p
+    @pType.set newRow, type
+    @pSize.set newRow, 0
+    @pTags.set newRow, Metakit::Bytes.new("", 0)
+    @pData.set newRow, Metakit::Bytes.new("", 0)
+    @vFentry.add newRow
+
+    if p != "/"
+      #
+      # Then, add an entry for the new file or directory in its parent.
+      #
+      dirName = File.basename(p) + "\0"
+      dataRef = @pData.ref(parentRow)
+      data = Metakit::Bytes.new(dirName, dirName.length)
+      size = @pSize.get(parentRow)
+      dataRef.modify(data, size, data.size)
+      @pSize.set(parentRow, size + data.size)
+    end
+
+    @storage.commit
+
+    fi = getFileIndex(p)
+    raise "[BUG] can't find new file: #{p}" if fi < 0
+
+    #
+    # Must use row obtained through @vFentry.
+    #
+    (@vFentry[fi])
+  end
+
+  class MkFile
+    attr_accessor :filePath, :fileRow, :filePos, :fileRead, :fileWrite
+
+    def initialize(path, fileRow, fpos, fread, fwrite)
+      @filePath  = path
+      @fileRow   = fileRow
+      @filePos   = fpos
+      @fileRead  = fread
+      @fileWrite = fwrite
+    end
+  end # class MkFile
+end # module MetakitFS
+
+module MkFileMod
+  def addTag(tag)
+    @fs.fs_fileTagAdd(@fobj, tag)
+  end
+
+  def deleteTag(tag)
+    @fs.fs_fileTagDelete(@fobj, tag)
+  end
+
+  def tags
+    @fs.fs_fileTags(@fobj)
+  end
+
+  def hasTagName?(tagName)
+    @fs.fs_fileHasTagName?(@fobj, tagName)
+  end
+
+  def hasTag?(tag)
+    @fs.fs_fileHasTag?(@fobj, tag)
+  end
+
+  def tagValues(tag)
+    @fs.fs_fileTagValues(@fobj, tag)
+  end
+end

data/lib/fs/MetakitFS/test/Makefile

@@ -0,0 +1,14 @@
+
+CC = c++
+
+CFLAGS = -I/usr/local/lib/ruby/1.8/i686-darwin8.10.1
+LDFLAGS = -static
+LIBS = -lmk4 -lruby
+
+mk4test: a.out
+	rm -f mk4test
+	cp a.out mk4test
+	ruby MkSelectFiles.rb collect_files.yaml mk4test
+	
+a.out: mk4test.c
+	$(CC) $(CFLAGS) $(LIBS) mk4test.c

data/lib/fs/MetakitFS/test/MkCollectFiles.rb

@@ -0,0 +1,165 @@
+require 'find'
+require 'fileutils'
+require 'yaml'
+require 'ostruct'
+require 'zlib'
+
+require 'fs/MiqFS/MiqFS'
+require 'fs/MetakitFS/MetakitFS'
+
+class MkCollectFiles
+  attr_writer :verbose
+
+  def initialize(collectionSpec, mkFile)
+    @csa = collectionSpec
+    @csa = YAML.load_file(collectionSpec) if @csa.kind_of? String
+    @csa = [@csa] if @csa.kind_of? Hash
+    raise "Invalid collection spec" unless @csa.kind_of? Array
+
+    dobj = OpenStruct.new
+    dobj.mkfile = mkFile
+    dobj.create = true
+    @mkFS = MiqFS.new(MetakitFS, dobj)
+
+    @verbose = false
+  end
+
+  def collect
+    @csa.each { |cs| doCollect(OpenStruct.new(cs)) }
+  end
+
+  def dumpSpec(specFile)
+    YAML.dump(@csa, File.open(specFile, "w"))
+  end
+
+  private
+
+  def doCollect(cs)
+    #
+    # The directory where the collection will be created.
+    #
+    destDir = cs.todir
+
+    #
+    # The directory relative to which the files will be collected.
+    #
+    Dir.chdir(cs.basedir)
+    puts "BASEDIR: #{Dir.pwd}" if @verbose
+
+    #
+    # Loop through the files and directories that are to be included in the collection.
+    #
+    cs.include.each do |i|
+      raise "File: #{i} does not exist" unless File.exist? i
+      #
+      # If this is a plain file, then include it in the collection.
+      #
+      unless File.directory? i
+        puts "FILE: #{i}" if @verbose
+        toFile = File.join(destDir, i)
+        makePath(File.dirname(toFile))
+        #
+        # If the file path matches an encrypt RE and doesn't
+        # match a noencrypt RE, then encrypt the contents of
+        # the file before copying it to the collection.
+        #
+        if cs.encrypt && cs.encrypt.detect { |e| i =~ e }
+          if !cs.noencrypt || !cs.noencrypt.detect { |ne| i =~ ne }
+            encryptFile(i, toFile)
+            next
+          end
+        end
+        #
+        # If the file path matches an compress RE and doesn't
+        # match a nocompress RE, then compress the contents of
+        # the file before copying it to the collection.
+        #
+        if cs.compress && cs.compress.detect { |e| i =~ e }
+          if !cs.nocompress || !cs.nocompress.detect { |ne| i =~ ne }
+            compressFile(i, toFile)
+            next
+          end
+        end
+        copyFile(i, toFile)
+        next
+      end
+
+      #
+      # If this is a directory, then recursively copy its contents
+      # to the collection directory.
+      #
+      puts "DIR: #{i}" if @verbose
+      Find.find(i) do |path|
+        #
+        # Prune directories that match an exclude RE.
+        #
+        if File.directory? path
+          Find.prune if cs.exclude && cs.exclude.detect { |e| path =~ e }
+          next
+        end
+        #
+        # Skip files that match an exclude RE.
+        #
+        next if cs.exclude && cs.exclude.detect { |e| path =~ e }
+        toFile = File.join(destDir, path)
+        makePath(File.dirname(toFile))
+        #
+        # If the file path matches an encrypt RE and doesn't
+        # match a noencrypt RE, then encrypt the contents of
+        # the file before copying it to the collection.
+        #
+        if cs.encrypt && cs.encrypt.detect { |e| path =~ e }
+          if !cs.noencrypt || !cs.noencrypt.detect { |ne| path =~ ne }
+            encryptFile(path, toFile)
+            next
+          end
+        end
+        #
+        # If the file path matches an compress RE and doesn't
+        # match a nocompress RE, then compress the contents of
+        # the file before copying it to the collection.
+        #
+        if cs.compress && cs.compress.detect { |e| path =~ e }
+          if !cs.nocompress || !cs.nocompress.detect { |ne| path =~ ne }
+            compressFile(path, toFile)
+            next
+          end
+        end
+        copyFile(path, toFile)
+      end
+    end if cs.include
+  end
+
+  def makePath(path)
+    return if @mkFS.fileExists? path
+    parentDir = @mkFS.fileDirname(path)
+    makePath(parentDir) unless @mkFS.fileExists? parentDir
+    @mkFS.dirMkdir(path)
+  end
+
+  def copyFile(src, dest)
+    puts "\t    COPY: #{src}\n\t      TO: #{dest}\n\n" if @verbose
+    File.open(src) do |ffo|
+      tfo = @mkFS.fileOpen(dest, "wb")
+      while (buf = ffo.read(4096))
+        tfo.write(buf, buf.length)
+      end
+      tfo.close
+    end
+  end
+  alias_method :encryptFile, :copyFile
+
+  def compressFile(src, dest)
+    puts "\tCOMPRESS: #{src}\n\t      TO: #{dest}\n\n" if @verbose
+    File.open(src) do |ffo|
+      tfo = @mkFS.fileOpen(dest, "wb")
+      zipper = Zlib::Deflate.new
+      while (buf = ffo.read(4096))
+        zipper << buf
+      end
+      tfo.write(zipper.deflate(nil, Zlib::FINISH))
+      tfo.addTag("compressed")
+      tfo.close
+    end
+  end
+end # class MkCollectFiles