complearn 0.6.2

data/ext/lib/CompLearnLib/CompressionObject.rb

@@ -0,0 +1,59 @@
+#
+#    CompressionObject
+#
+#    This is a compressible object. Abstracts string/files
+#
+#    $Id: CompressionObject.rb,v 1.1 2003/10/23 17:03:59 cilibrar Exp $
+#
+
+class CompressionObject
+    
+    private_class_method :new
+
+    #
+    # private constructor
+    #
+    def initialize(str,id,isFile)
+        @cId = id
+        @str = str
+        @isFile = isFile
+    end
+
+    #
+    # Initialize from file
+    #
+    def self.fromFile(fname)
+        c = new('',fname,true)
+        return(c)
+    end
+
+    #
+    # Initialize from string
+    #
+    def self.fromString(str,id)
+        c = new(str,id,false)
+        return(c)
+    end
+
+    #
+    # id is to_s
+    #
+    def to_s()
+        @cId
+    end
+
+    #
+    # return the value depending on whether its a file or
+    # string
+    #
+    def value()
+        if @isFile
+            raise "Cannot find #{@cId}" unless File.exist?(@cId)
+            raise "Cannot read #{@cId}" unless File.stat(@cId).readable?
+            return(File.open(@cId,"r").read)
+        else
+            return(@str)
+        end
+    end
+
+end

data/ext/lib/CompLearnLib/CompressionTask.rb

@@ -0,0 +1,99 @@
+#
+#    CompressionTask
+#    
+#    task object to handle file compressions, used
+#    with TaskMaster
+#
+#    $Id: CompressionTask.rb,v 1.3 2003/11/14 14:51:44 cilibrar Exp $
+
+require 'CompLearnLib/Task.rb'
+require 'CompLearnLib/CLConfig.rb'
+require 'CompLearnLib/FoundComp.rb'
+
+if FoundComp::FOUNDBZIP2
+  require 'bz2'
+end
+
+if FoundComp::FOUNDGZIP
+  require 'zlib'
+end
+
+include MTask
+
+def gzipCompress(str)
+  level = Zlib::BEST_COMPRESSION
+  z = Zlib::Deflate.new(level)
+  dst = z.deflate(str, Zlib::FINISH)
+  z.close
+  dst
+end
+
+class CompressionTask < Task
+
+  attr_reader :idTotal
+
+  def initialize(cObjects)
+    @cObjects = cObjects.clone
+    @idTotal = ""
+    cObjects.each { |obj|
+        idTotal << obj.to_s
+    }
+  end
+
+  def execute()
+    total = ""
+    @cObjects.each { |obj|
+      total << obj.value
+    }
+    compmethod = CLConfig.getDefaultConfig().compressor
+
+    case compmethod.downcase
+
+      when "bz2", "bzip2", "bzip", "bz"
+
+      if FoundComp::FOUNDBZIP2
+        reply(BZ2.compress(total).size)
+      else
+        fail "Sorry, bzip2 compression is not installed on this system."
+      end
+
+      when "gzip", "gz"
+      if FoundComp::FOUNDGZIP
+        reply(gzipCompress(total).size)
+      else
+        fail "Sorry, gzip compression is not installed on this system."
+      end
+
+      when "shell"
+        cmd = CLConfig.getDefaultConfig().compressorCommand
+        result = 0
+        begin
+        f = IO.popen(cmd, "w+")
+        writer = Thread.new() {
+          blocksize, inpos = 1024, 0
+          while inpos < total.size
+            wanted = total.size - inpos
+            wanted = blocksize if wanted > blocksize
+            inpos += f.write(total[inpos...inpos+wanted])
+          end
+          f.close_write
+        }
+        result = f.read.size
+        f.close
+        rescue
+          puts "Got error: #{$!}"
+          result = -1
+        end
+        reply(result)
+
+      when "none"
+        fail "Sorry, no valid compressors installed!"
+
+      else
+        fail "Sorry, compression method #{compmethod} is not supported"
+
+    end
+
+  end
+end
+

data/ext/lib/CompLearnLib/DistMatrix.rb

@@ -0,0 +1,18 @@
+class DistMatrix
+
+  def DistMatrix.readFromFile(f)
+    f = File.open(f, 'r') if f.is_a?(String)
+    result = f.readlines
+    result.delete_if { |line| line =~ /^#/ || line !~ /\S+/ }
+    result.collect { |line| line.split(' ').collect { |i| i.to_f } }
+  end
+
+  def DistMatrix.convertToString(distmatrix)
+    result = ''
+    distmatrix.each { |row|
+      result << row.join(' ') + "\n"
+    }
+    result
+  end
+
+end

data/ext/lib/CompLearnLib/FoundComp.rb

@@ -0,0 +1,10 @@
+class FoundComp
+  FOUNDGZIP = true
+  FOUNDBZIP2 = true
+  VERSION = '0.6.2'
+  def FoundComp.defaultCompressor()
+    return "bzip2" if FOUNDBZIP2
+    return "gzip" if FOUNDGZIP
+    return "none"
+  end
+end

data/ext/lib/CompLearnLib/FoundComp.rb.in

@@ -0,0 +1,10 @@
+class FoundComp
+  FOUNDGZIP = @FOUNDGZIP@
+  FOUNDBZIP2 = @FOUNDBZIP2@
+  VERSION = '@VERSION@'
+  def FoundComp.defaultCompressor()
+    return "bzip2" if FOUNDBZIP2
+    return "gzip" if FOUNDGZIP
+    return "none"
+  end
+end

data/ext/lib/CompLearnLib/Ncd.rb

@@ -0,0 +1,248 @@
+#
+#    Ncd calculations
+#
+#    $Id: Ncd.rb,v 1.2 2003/10/30 18:03:21 cilibrar Exp $
+#
+
+require 'CompLearnLib/CLConfig'
+require 'CompLearnLib/CompressionObject'
+require 'CompLearnLib/CompressionTask'
+require 'CompLearnLib/Task'
+include MTask
+
+class Ncd
+
+    #private_class_method :ncd
+
+    #
+    #    
+    def initialize()
+        @config = CLConfig.getDefaultConfig()
+        # for now
+    end
+   
+    ### private  
+   
+    # 
+    # get the ncd given pre compressed input
+    # 
+    # ca:        size of compressed a
+    # cb:        size of compressed b
+    # cab:       size of compressed ab
+    # cba:       size of compressed ba
+    #
+    # returns: ncd of a and b
+    #
+    def ncd(ca,cb,cab,cba)
+        maxk = ca < cb ? cb : ca
+        kab = cba - cb
+        kba = cab - ca
+        maxck = kab < kba ? kba : kab
+        (maxck.to_f / maxk.to_f).to_f
+    end
+
+    #
+    # execute the list of given compression tasks
+    #
+    # list:    a list of objects to compress of the
+    #          form [[object1],[object1,object2],..]
+    #
+    # returns: a hash of compressed sizes, the key to
+    #          the hash is the id (or concatenation of ids)
+    #          of the compressible object(s)
+    #
+    def compress(list)
+
+        # dispatch compression tasks
+        results = { }
+        todo = list.size
+        TaskMaster.init
+        list.each { |item|
+            t = CompressionTask.new(item)
+            TaskMaster.enqueue(t) { |res, t, src|
+                # XXX put in resv chr
+                results[t.idTotal] = res
+                todo -= 1
+            } 
+        }
+
+        # harvest results
+        while(todo != 0)
+            TaskMaster.waitForReply
+        end
+
+        # add vice versa compression if symmetric
+        if @config.isSymmetric?
+            list.each { |item|
+                if item.size > 1
+                    abName = item.join()
+                    newItem = item.reverse()
+                    baName = newItem.join()
+                    results[baName] = results[abName]
+                end
+            }
+        end
+                    
+        return(results)
+    end
+
+    #
+    # build singles and pairs to compress from the
+    # given list
+    #
+    def buildPairs(list)
+        cList = Array.new()
+        list.each { |item|
+            cList.push([item])
+            if @config.isSymmetric?
+                list.each { |item2|
+                    cList.push([item,item2])
+                }
+            else
+                list.each { |item2|
+                    cList.push([item,item2])
+                    cList.push([item2,item])
+                }
+            end
+        }
+        return(cList)
+    end
+
+    #
+    # build singles and pairs with the given object 
+    #
+    def buildSinglePairs(a,list)
+        cList = Array.new()
+        list.each { |item|
+            cList.push([item])
+            if @config.isSymmetric?
+                cList.push([a,item])
+            else
+                cList.push([a,item])
+                cList.push([item,a])
+            end
+        }
+        return(cList)
+    end
+
+    ### single pair ncd
+
+    #
+    # calculate ncd from two cObjects
+    #
+    def ncdSingle(a,b)
+        if @config.isSymmetric?
+            c = compress([[a],[b],[a,b]])
+            c["#{b}#{a}"] = c["#{a}#{b}"]
+        else
+            c = compress([[a],[b],[a,b],[b,a]])
+        end
+        ncd(c["#{a}"],c["#{b}"],c["#{a}#{b}"],c["#{b}#{a}"])
+    end
+
+    #
+    # calculate ncd from 2 strings
+    #
+    def ncdString(a, b)
+        aObj = CompressionObject.fromString(a, a)
+        bObj = CompressionObject.fromString(b, b)
+        ncdSingle(aObj,bObj)
+    end
+
+    #
+    # calculate ncd from 2 files
+    #
+    def ncdFile(a, b)
+        aObj = CompressionObject.fromFile(a)
+        bObj = CompressionObject.fromFile(b)
+        ncdSingle(aObj,bObj)
+    end
+
+    ### vector ncd
+
+    #
+    # calculate the ncd between one object and a list
+    # of others
+    #   
+    def ncdVector(a,list)
+
+        # precompress everything necessary
+        cList = buildSinglePairs(a,list)
+        cList.push([a])
+        c = compress(cList)
+
+        # calculate ncd of a with everything else
+        res = Array.new()
+        list.each { |b|
+            res.push(ncd(c["#{a}"],c["#{b}"],c["#{a}#{b}"],c["#{b}#{a}"]))
+        }
+        return(res)
+    end 
+
+    #
+    # calculate the mutual ncd of a string with a list of strings
+    #
+    def ncdVectorString(a, list)
+        aObj = CompressionObject.fromString(a)
+        listObj = list.collect { |item|
+                    CompressionObject.fromString(item)
+                  }
+        ncdVector(aObj,listObj)
+    end
+
+    #
+    # calculate the mutual ncd of a file with a list of files
+    #
+
+    def ncdVectorFile(a, list)
+        aObj = CompressionObject.fromFile(a)
+        listObj = list.collect { |item|
+                    CompressionObject.fromFile(item)
+                  }
+        ncdVector(aObj,listObj) 
+    end
+
+
+    ### matrix ncd
+
+    #
+    # calculate the mutual ncd of all lists objects
+    #
+    def ncdMatrix(list)
+        # precompress everything necessary
+        cList = buildPairs(list)
+        c = compress(cList)
+
+        # calculate ncd of a with everything else
+        res = Array.new()
+        list.each { |a|
+            res << [ ]
+            list.each { |b| 
+                res[-1] << ncd(c["#{a}"],c["#{b}"],c["#{a}#{b}"],c["#{b}#{a}"])
+            }
+        }
+        return(res)
+    end
+
+    #
+    # calculate the mutual ncd of a list of strings
+    #
+    def ncdMatrixString(list)  # most caching benefit
+        listObj = list.collect { |item|
+            CompressionObject.fromString(item)
+        }    
+        ncdMatrix(listObj)
+    end
+
+    #
+    # calculate the mutual ncd of a list of files
+    #
+    def ncdMatrixFile(list)
+        listObj = list.collect { |item|
+            CompressionObject.fromFile(item)
+        }    
+        ncdMatrix(listObj)
+    end
+end
+
+