complearn 0.6.2

data/ext/TreeScore.c

@@ -0,0 +1,244 @@
+#include <assert.h>
+#include "TreeScore.h"
+#include "Quartet.h"
+#include <malloc.h>
+#include "ruby.h"
+
+#ifndef VALUEFUNC
+#define VALUEFUNC(x) RUBY_METHOD_FUNC(x)
+#endif
+
+static VALUE cTreeScore;
+
+/*
+ * Holds a list of quartets with associated weights.
+ * Keeps track of worst and best possible scores using this list.
+ */
+struct TreeScore {
+  weight_t worst, best;
+  weight_t penalty;
+  struct QuartetList *ql;
+};
+
+/*
+ * Allocate a new TreeScore
+ */
+struct TreeScore *newTreeScore()
+{
+  // m11.
+  struct TreeScore *result = calloc(sizeof(struct TreeScore), 1);
+  result->penalty = 0.0;
+  return result;
+}
+/*
+ * Free a C/C++ TreeScore object
+ */
+static void ts_free(void *vts)
+{
+  freeTreeScore((struct TreeScore *) vts);
+}
+
+void freeTreeScore(struct TreeScore *ts)
+{
+  // f11.
+  freeQuartetList(ts->ql);
+  ts->ql = 0;
+  free(ts);
+}
+
+
+/*
+ * Convert a Ruby Tree into a C/C++ FastTree pointer 
+ */
+struct FastTree *convertTreeFromRuby(VALUE tree)
+{
+  VALUE edges = rb_iv_get(tree, "@edges");
+  int translation[MAXNODES];
+  int nodeCount = RARRAY(edges)->len;
+  int speciesCount = (nodeCount + 2) / 2;
+  struct FastTree *ft = newFastTree(nodeCount);
+  int currentKernel=0, currentSpecies=0;
+  int i, j;
+  ft->spec = speciesCount;
+  for (i = 0; i < nodeCount; ++i) {
+    VALUE neighbors = rb_ary_entry(edges, i);
+    int ns = RARRAY(neighbors)->len;
+    switch (ns) {
+      case 1:
+        translation[i] = currentSpecies++;
+        break;
+      case 3:
+        translation[i] = speciesCount + currentKernel++;
+        break;
+      default:
+        assert("Bad tree!" && 0);
+    }
+  }
+  for (i = 0; i < nodeCount; ++i) {
+    VALUE neighbors = rb_ary_entry(edges, i);
+    int ns = RARRAY(neighbors)->len;
+    for (j = 0; j < ns; ++j) {
+      VALUE curn = rb_ary_entry(neighbors, j);
+      push_back(&ft->tree->nl[translation[i]], translation[NUM2INT(curn)]);
+    }
+  }
+  ft->spm = allShortestPathTrees(ft->tree);
+  return ft;
+}
+
+/*
+ * Convert a Ruby 2-dimensional array into a C/C++ DistMatrix
+ */
+struct DistMatrix *convertDistMatrixFromRuby(VALUE dm)
+{
+  int size = RARRAY(dm)->len;
+  struct DistMatrix *rdm = newDistMatrix(size);
+  int i, j;
+  for (i = 0; i < size; ++i) {
+    for (j = 0; j < size; ++j) {
+      weight_t val = NUM2DBL(rb_ary_entry(rb_ary_entry(dm, i),j));
+      rdm->vals[i][j] = val;
+    }
+  }
+  return rdm;
+}
+
+/*
+ * Makes a list consisting of only the best quartets (1/3 as many as full)
+ */
+static VALUE ts_makeBest(VALUE cl, VALUE dm)
+{
+  struct TreeScore *ts = newTreeScore();
+  VALUE tdata = Data_Wrap_Struct(cl, 0, ts_free, ts);
+  struct DistMatrix *gdm = convertDistMatrixFromRuby(dm);
+  ts->ql = makeBestQuartetList(gdm, &ts->worst, &ts->best);
+  freeDistMatrix(gdm);
+  return tdata;
+}
+
+/*
+ * Makes a list of every quartet with corresponding cost.
+ */
+static VALUE ts_makeFull(VALUE cl, VALUE dm)
+{
+  struct TreeScore *ts = newTreeScore();
+  VALUE tdata = Data_Wrap_Struct(cl, 0, ts_free, ts);
+  struct DistMatrix *gdm = convertDistMatrixFromRuby(dm);
+  ts->ql = makeFullQuartetList(gdm, &ts->worst, &ts->best);
+  freeDistMatrix(gdm);
+  return tdata;
+}
+
+/*
+ * Return the worst possible tree cost total.  This is used to calculate S(T).
+ */
+static VALUE ts_worst(VALUE self)
+{
+  struct TreeScore *ts;
+  Data_Get_Struct(self, struct TreeScore, ts);
+  return rb_float_new(ts->worst);
+}
+
+/*
+ * Return the best possible tree cost total. This is used to calculate S(T).
+ */
+static VALUE ts_best(VALUE self)
+{
+  struct TreeScore *ts;
+  Data_Get_Struct(self, struct TreeScore, ts);
+  return rb_float_new(ts->best);
+}
+
+/*
+ * Returns the number of neighbors who are leaves to a given node
+ */
+int countLeafNeighbors(const struct FastTree *ft, qbase_t which)
+{
+  int nc = ft->tree->nl[which].size;
+  int i;
+  int leafCount = 0;
+  for (i = 0; i < nc; ++i)
+    if (ft->tree->nl[ft->tree->nl[which].n[i]].size == 1)
+      leafCount++;
+  return leafCount;
+}
+
+/*
+ * Returns the number of nodes that are unpaired
+ */
+int countBadNodes(const struct FastTree *ft)
+{
+  int i;
+  int badCount = 0;
+  for (i = ft->spec; i < ft->tree->size; ++i)
+    if (countLeafNeighbors(ft, i) == 1)
+      badCount++;
+  return badCount;
+}
+
+static VALUE ts_penaltyeq(VALUE self, VALUE val)
+{
+  struct TreeScore *ts;
+  Data_Get_Struct(self, struct TreeScore, ts);
+  ts->penalty = NUM2DBL(val);
+  return rb_float_new(ts->penalty);
+}
+
+static VALUE ts_penalty(VALUE self)
+{
+  struct TreeScore *ts;
+  Data_Get_Struct(self, struct TreeScore, ts);
+  return rb_float_new(ts->penalty);
+}
+
+/*
+ * Return a tree's S(T) score, between 0.0 (worst) and 1.0 (best), (penalty may force it outside range)
+ */
+void printFastTree(struct FastTree *ft)
+{
+  int i, j;
+  printf("Tree size: %d\n", ft->tree->size);
+  for (i = 0; i < ft->tree->size; ++i) {
+    const struct NodeList *cur = &ft->tree->nl[i];
+    printf("Node %d: %d neighbors, %d leaf-neighbors: ", i, ft->tree->nl[i].size, countLeafNeighbors(ft, i));
+    for (j = 0; j < cur->size; ++j) {
+      printf("%d->%d, ", i, cur->n[j]);
+    }
+    printf("\n");
+  }
+  printf("\n");
+}
+
+static VALUE ts_score(VALUE self, VALUE tree)
+{
+  int badNodeCount;
+  struct TreeScore *ts;
+  struct FastTree *ft = convertTreeFromRuby(tree);
+  weight_t score;
+	float result, radj;
+  Data_Get_Struct(self, struct TreeScore, ts);
+  score = calculateWeightedScore(ft, ts->ql);
+  result = (score - ts->worst) / (ts->best - ts->worst);
+  badNodeCount = countBadNodes(ft);
+  radj = - ts->penalty * countBadNodes(ft);
+  freeFastTree(ft);
+  return rb_float_new(result + radj);
+}
+
+void initTreeScore()
+{
+  cTreeScore = rb_define_class("TreeScore", rb_cObject);
+  rb_define_singleton_method(cTreeScore, "makeBestList", VALUEFUNC(ts_makeBest), 1);
+  rb_define_singleton_method(cTreeScore, "makeFullList", VALUEFUNC(ts_makeFull), 1);
+  rb_define_method(cTreeScore, "penalty", VALUEFUNC(ts_penalty), 0);
+  rb_define_method(cTreeScore, "penalty=", VALUEFUNC(ts_penaltyeq), 1);
+  rb_define_method(cTreeScore, "score", VALUEFUNC(ts_score), 1);
+  rb_define_method(cTreeScore, "worst", VALUEFUNC(ts_worst), 0);
+  rb_define_method(cTreeScore, "best", VALUEFUNC(ts_best), 0);
+}
+
+void Init_CompLearn()
+{
+  initTreeScore();
+}
+

data/ext/TreeScore.h

@@ -0,0 +1,3 @@
+struct TreeScore;
+void initTreeScore();
+void freeTreeScore(struct TreeScore *ts);

data/ext/config.h

@@ -0,0 +1,65 @@
+/* ext/config.h.  Generated by configure.  */
+/* ext/config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if stdbool.h conforms to C99. */
+/* #undef HAVE_STDBOOL_H */
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if the system has the type `_Bool'. */
+/* #undef HAVE__BOOL */
+
+/* Name of package */
+#define PACKAGE "complearn"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME ""
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING ""
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION ""
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Version number of package */
+#define VERSION "0.6.2"
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */

data/ext/config.h.in

@@ -0,0 +1,64 @@
+/* ext/config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if stdbool.h conforms to C99. */
+#undef HAVE_STDBOOL_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#undef HAVE_SYS_TIME_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to 1 if the system has the type `_Bool'. */
+#undef HAVE__BOOL
+
+/* Name of package */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Version number of package */
+#undef VERSION
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const

data/ext/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+#have_library('bz2')
+create_makefile('CompLearn')

data/ext/lib/CompLearnLib/CLConfig.rb

@@ -0,0 +1,241 @@
+#
+#    CLConfig
+#
+#    Reads a given configuration file of the following form:
+#
+#    Compressor:    [BZ2,..]
+#    InputDir:      [input directory]
+#    OutputDir:     [output directory]
+#    WorkDir:       [working directory]
+#    Symmetric:     [yes/no]
+#    Hosts:         [list of hosts, seperated by ,'s]
+#
+#    Comments in the configuration file are prefixed with #
+#    individual values are accessed by configObj.[value]
+#    Note: this will exit if an error is encountered
+#
+#    $Id: CLConfig.rb,v 1.5 2003/11/27 16:13:46 cilibrar Exp $
+#
+
+require 'yaml'
+require 'CompLearnLib/FoundComp.rb'
+
+class CLConfig
+
+# The allowed config variables
+
+    @@BASECONFIGVARS = 
+[     #  prettyName     isArray   Type    Default Value
+      [ 'Compressor'    , false,  String,     FoundComp.defaultCompressor() ],
+      [ 'CompressorCommand'   , false,  String,     'gzip -c -' ],
+      [ 'InputDir'      , false,  String,     'in'        ],
+      [ 'OutputDir'     , false,  String,     'out'       ],
+      [ 'WorkDir'       , false,  String,     'work'      ],
+      [ 'Symmetric'     , false,  TrueClass,   true       ],
+      [ 'Hosts'         , true,   String,  ['localhost']  ],
+      [ 'SingleProcess' , false,  TrueClass,   true      ],
+# maketree
+      [ 'UnpairedPenalty',false, Float, 0.0 ],
+      [ 'UseBestThirdOnly', false, TrueClass, false ],
+      [ 'MaxFailedTries', false, Integer, 100 ],
+      [ 'TreesPerTry', false, Integer, 1000 ],
+      [ 'InternalNodePrefix', false, String, 'n' ],
+]
+
+  # A class-method  (since self is in class/module-scope, it refers to
+  # the Class object CLConfig).  This one just tries to find a configuration
+  # file in a default spot, either in $HOME or /etc
+  # Also, this method caches the first read config object as a Singleton.
+
+  def self.singleUser?()
+    defined?(@@SINGLEUSER)
+  end
+  def self.setSingleUser()
+    @@SINGLEUSER = true
+  end
+  def self.setDefaultConfig(cfg)
+    @@DEFCON = cfg
+  end
+  def self.getDefaultConfig
+    unless defined?(@@DEFCON)
+      homedir = ENV['HOME'] || '/home'
+      filename = 'complearnrc'
+      goodcon = nil
+      [ "#{homedir}/.#{filename}", "/etc/#{filename}" ].each { |pathname|
+        next unless File.exist?(pathname)
+        goodcon = CLConfig.new(pathname)
+        break if goodcon
+      }
+      goodcon = CLConfig.new(nil) unless goodcon
+      @@DEFCON = goodcon
+    end
+    @@DEFCON
+  end
+    @@CONFIGVARS = { }
+    @@BASECONFIGVARS.each { | prettyName, isArray, typ, defval |
+      lowerName = prettyName.downcase
+      methodName = prettyName[0..0].downcase + prettyName[1..-1]
+      #    accessing methods
+      if methodName == 'hosts'
+        def hosts() (ARGV.include?('-s') || CLConfig.singleUser?) ?
+                          [@hosts[0]]    :  @hosts
+        end
+      else
+        module_eval "def #{methodName}() @#{methodName} end"
+      end
+      module_eval "def is#{prettyName}?() #{methodName} end" if typ == TrueClass
+      @@CONFIGVARS[lowerName] = [ isArray, typ, defval, prettyName, methodName ]
+    }
+    #
+    # constructor, read file
+    #
+    def initialize(filename)
+        @@CONFIGVARS.each_value { |a|
+          methodName, defVal = a[4], a[2]
+          instance_eval("@#{methodName} = defVal")
+        }
+        if filename
+          @filename = filename
+          readCLConfig()
+        end
+    end
+
+    def self.writeDefaultConfigFile(fname)
+      cfgmap = { }
+      @@CONFIGVARS.each { |lowername, stuff|
+        isArray, typ, defval, prettyName, methodName = stuff
+        cfgmap[prettyName] = defval
+      }
+      File.open(fname, "w") { |f|
+        f.write(cfgmap.to_yaml)
+      }
+    end
+
+    #
+    # read in the configuration file
+    #
+    def readCLConfig()
+        begin
+            yamlcfg = YAML::parse(File::new(@filename,"r").read)
+        rescue
+            print "Unable to read config file " + @filename + ": " + $!   
+            print "\n"
+            exit
+        end
+
+        #fail "@filename must be a map not a #{yamlcfg.type_id}" unless yamlcfg.type_id.to_s == 'map'
+
+        yamlcfg.value.each { |itagname, whatnot|
+              tagname, value = itagname.downcase, whatnot[1].value
+              if @@CONFIGVARS.has_key?(tagname)
+                isArray,typ,default, pretty, methodName = @@CONFIGVARS[tagname]
+                if isArray
+                  result = value.map { |i| parseValue(typ, i.value) }
+                else
+                  result = parseValue(typ, value)
+                end
+                instance_eval("@#{methodName}=result")
+              else
+                fail "Unknown configuration option: #{itagname}"
+              end
+        }
+    end
+
+    def parseValue(typ, val)
+      val = val.clone
+      # remove whitespace
+      val.gsub!(/^\s+/,'')
+      val.gsub!(/\s+$/,'')
+      case typ.to_s   # to_s necessary because === checks is_a? for Class
+
+        when 'TrueClass'
+          return val =~ /true/i || val =~ /yes/i || val =~ /on/i
+
+        when 'String'
+          return val 
+
+        when 'Float'
+          return val.to_f
+
+        when 'Integer'
+          return val.to_i
+
+        else
+          puts "Illegal type: #{typ} for value #{val}"
+          exit(1)
+
+      end
+        
+    end
+
+  # Searches for a file or directory specified by fname
+  # if fname is relative, it will search for it in the following
+  # order:
+  # inputDir specified in this CLConfig object
+  # current working directory
+  #
+  # If found, a string is returned with the absolute (full) pathname.
+  # If not, an exception is raised
+    def findInputFile(fname)
+      raise "fname can not be nil" if fname == nil
+      maybe = File.expand_path(fname, self.inputDir)
+      return maybe if File.exist?(maybe)
+      maybe = File.expand_path(fname)
+      return maybe if File.exist?(maybe)
+      raise "Cannot find file to open: #{fname}"
+    end
+
+    def readTaggedFileList(fname)
+      training = [ ]
+      features = [ ]
+      testing = [ ]
+      f = File.open(fname, 'r')
+      while line = f.gets
+        line.chomp!
+        next if line =~ /^\s*#/
+        next unless line =~ /[\S]/
+        if line =~ /^\s*(\S+)\s+(\S)\s+([^\s#]+)/
+          tagnum, tagtype, fname = $1.to_i, $2.downcase, $3
+          realfname = findInputFile(fname)
+          if tagtype == 'p'
+            testing << [ tagnum.to_i, realfname ]
+          elsif tagtype == 'f'
+            features << realfname
+          else   # tagtype == 'g'
+            training << [ tagnum.to_i, realfname ]
+          end
+        end
+      end
+      return [ training, features, testing ]
+    end
+    def getFilelistFromDirOrFile(objname)
+      files = [ ]
+      if File.ftype(objname) == 'directory'
+        d = Dir.new(objname)
+        d.each { |f|
+          goodf = "#{objname}/#{f}"
+          files << goodf if File.ftype(goodf) == 'file'
+        }
+        files.sort!
+      else
+        files = readFileList(objname)
+      end
+      files
+    end
+    def readFileList(fname)
+      f = File.open(fname, 'r')
+      result = [ ]
+      while line = f.gets
+        line.chomp!
+        next if line =~ /^#/
+        next unless line =~ /[\S]/
+        realfname = findInputFile(line)
+        result << realfname
+      end
+      result
+    end
+    def self.printVersionAndExit()
+      puts "CompLearn #{FoundComp::VERSION}"
+      exit(0)
+    end
+end