svmlab 1.0.0

data/lib/svmfeature.rb

@@ -0,0 +1,337 @@
+require 'yaml'
+#require 'forkoff'
+require 'svmlab-config.rb'
+
+# An SVMFeature object is initialized with either a string in YAML format
+# or a file object pointing to a file with configuration given in YAML format.
+#
+# The SVMFeature class is the 'middleware' between software that calculates
+# features and the SVMLab class. SVMFeature's features are :
+# - Maintanence of a database of calculated features. This is done to
+#   minimize CPU time needed when developing an SVM experiment. The motto
+#   is that a calculation should be done only once and then never again.
+# - Meta features can be defined that uses other features to calculate
+#   a feature . In these cases, a structure similar to the entire configuration
+#   should be given for this feature.
+#
+# ---CONFIGURATION---
+#
+# SVMFeature is initiated by a configuration file in YAML format.
+# Required fields in configuration are:
+# (all paths can be given either absolute or relative to BaseDir)
+#
+# ---
+# Features:
+# - <targetfeature> 
+# - <feature1>
+# - <feature2>
+# ...
+# BaseDir: <base directory>
+# DataSet: <path of file giving the dataset>
+#           OR
+#          <prefix of a set of files giving the dataset>
+# Example: If Dataset is not given, this gives name(s) of examples
+#          to use.
+# Groups: <range (n1..n2) or (n1...n2) in example name to use for grouping>
+#          OR
+#         <file prefix relative to BaseDir for files giving groups>
+# Methods: <path of .rb file holding all feature calculation methods>
+# --------------------------------------------------------------
+# targetfeature:
+#   HomeDir: <home directory>
+#       If HomeDir is not given, it will be set to BaseDir/featurename
+#   Method: <the method calculating this feature>
+#       If Method is not given, an attempts is made to acquire
+#       the feature from the database. If it fails, ERROR is reported.
+#   Dimensions: <the number of dimensions in this feature>
+#       If Dimensions is not given, it will be assumed to be 1
+#       and only the first value for each example will be used
+#   <Further specific configuration of this feature>
+# feature1:
+#   HomeDir: <home directory>
+#   Method: <the method calculating this feature>
+#   Dimensions: <the number of dimensions in this feature>
+#   <Further specific configuration of this feature>
+# feature2:
+# ...
+# featureN:
+#    If featureN configuration is not given, then all default settings
+#    will apply to this feature.
+# ...
+#
+#
+
+class SVMFeature < Hash
+
+  attr_reader :dim
+
+  # config is either a file object or a string object.
+  def initialize(config)
+    @config = SVMFeaturesConfig.new(config)
+    #Get examples
+    @examples = 
+      if dataset = @config['DataSet']
+        dir = dataset.split(/\//)[0...-1].join('/')+'/'
+        if File::file?(dataset)
+          open(dataset) { |f| f.read }.split
+        elsif (files = Dir::entries(dir).
+               grep(/^#{dataset.split(/\//).last}/)).
+               size>0
+          files.inject([]) { |exarray,fname| 
+            exarray += open(dir+fname){|f| f.read}.split }          
+        end
+      elsif example = @config['Example']
+        if example.is_a? Array
+          example
+        else
+          [ example ]
+        end
+      end
+    # Set @feature to an empty hash
+    @feature = {}
+    #tmparr = @examples.forkoff :processes => 1 do |ex|
+    #tmparr = @examples.forkmap 1 do |ex|
+    tmparr = @examples.map do |ex|
+      begin
+        getExAllFeatures(ex)
+      rescue
+        STDERR.puts $!
+        $!
+      end
+    end
+    @examples.zip(tmparr).each do |k,v|
+      self[k] = v
+    end
+  end
+
+  # Returns a string giving the path to the file holding
+  # the feature for the current settings.
+  # Returns nil if current setting do not match any file.
+  def getDataFile(feature)
+    dfile = nil
+    filemapfname = @config[feature]['HomeDir'] + 'filemap.yml'
+    return nil if !File.file?(filemapfname)
+    open(filemapfname,'r') do |f|
+      if f.flock(File::LOCK_SH) 
+        begin
+          filemap = YAML.load(f)
+          if cfg = filemap.find {|c| c['Config'] == @config[feature]}
+            dfile = @config[feature]['HomeDir'] + cfg['FeatureFile'] 
+          else
+            dfile = nil
+          end
+        ensure
+          f.flock(File::LOCK_UN)
+        end
+      end
+    end
+    dfile
+  end
+
+  # Returns an array of floats giving the selected feature of the selected example.
+  def getExFeature(example, feature)
+    x = getExFeatureInternal(example, feature)
+    if !x then raise "ERROR: #{feature} is nil." end
+    if x =~ /^ERROR/
+      raise "ERROR (#{feature}): #{x.split[1..-1].join(' ')}"
+    end
+    if (dim=x.split.size) != @config[feature]['Dimensions']
+      raise "ERROR (#{feature}): Number of dimensions (#{dim})" + 
+            " for #{example} is not correct"
+    end
+    x.split.map{ |v| Float(v)}
+  end
+
+  # Returns a string giving the selected feature of the selected example.
+  def getExFeatureInternal(example,feature)
+    val = nil
+
+    # 0. Check the hash
+    if @feature[feature] and @feature[feature][example]
+      return @feature[feature][example]
+    end
+
+    # 1. Look in database if value is available
+    if dfile = getDataFile(feature)
+      open(dfile,'r') do |f|
+        if f.flock(File::LOCK_SH) 
+          begin
+            val = if @feature[feature] = YAML.load(f) 
+                  then @feature[feature][example]
+                  else nil end
+          ensure
+            f.flock(File::LOCK_UN)
+          end
+        end
+      end
+    end
+    return val if val
+
+    # 2. Calculate the value
+    calhash = {}
+    begin
+      method = @config[feature]['Method']
+      raise ArgumentError, "No method given for calculation" if !method
+      calhash = eval("#{@config[feature]['Method']}(@config[feature],example)")
+      if !calhash.is_a? Hash
+        raise "Incorrect output format from #{@config[feature]['Method']}"
+      end
+      calhash.each do |k,v|
+        if !v.is_a? String
+          raise "Incorrect output class (#{v.class})" +
+                "from #{@config[feature]['Method']}"
+        end
+      end
+    rescue ArgumentError
+      raise 
+    rescue NameError
+      raise NameError, "Method #{method} not found."
+    rescue
+      error = 'ERROR:' + $!.to_s.split(/\n/).shift.split(/\:/).pop
+      calhash = {example => error}
+    end
+    if !calhash or !calhash[example]
+      calhash[example] = 'ERROR: No output from method'
+    end
+    val = calhash[example]
+
+    # Update filemap.yml
+    if !getDataFile(feature)
+      filemapfname = @config[feature]['HomeDir'] + 'filemap.yml'
+      open(filemapfname,'w') {} if !File.file?(filemapfname)
+      open(filemapfname,'r+') do |f|
+        if f.flock(File::LOCK_EX) 
+          begin
+            filemap = if tmp = YAML.load(f) then tmp
+                      else [] end
+            datafile = feature + filemap.size.to_s + '.yml'
+            filemap.push({'FeatureFile' => datafile,
+                           'Config'     => @config[feature] })
+            f.rewind
+            YAML.dump(filemap,f)
+          ensure
+            f.flock(File::LOCK_UN)
+          end
+        end
+      end
+    end
+
+    # Add all outcome of calculation to the database
+    dfile = getDataFile(feature)
+    open(dfile,'a+') do |f|
+      if f.flock(File::LOCK_EX) 
+        begin
+          oldhash = YAML.load(f)
+          f.puts '--- ' if !oldhash 
+          calhash.each do |k,v|
+            if !oldhash or !oldhash[k]
+              h = {k => v}
+              f.puts h.to_yaml[5..-1]
+            end
+          end
+        ensure
+          f.flock(File::LOCK_UN)
+        end
+      end
+    end
+    puts "#{DateTime.now}, Calculated #{example}, #{feature} = #{val}"
+    val
+  end
+  
+  # Returns an array of floats containing all features for given example
+  def getExAllFeatures(example)
+    @config['Features'].inject([]) do |array,feature|
+      f = getExFeature(example, feature)
+      if array.empty? and (c=@config['PosClassFrom'])
+        f.first >= c ? [1] : [-1]
+      else
+        array + f 
+      end
+    end
+  end
+  
+  # Returns a hash for all examples in the data set
+  #     key : example name
+  #     value : array of values (float)
+  # getAllFeatures is dependent on a data set being given in the configuration
+  def getAllFeatures()
+    @examples.inject({}) { |output,example|
+      begin
+        output[example] = getExAllFeatures(example)
+      rescue
+        #puts "Excluding #{example}"
+      end
+      output
+    }
+  end
+
+  # Prints all examples and their features
+  # Prints to a file if given, otherwise to standard output.
+  def printFeatures(file = nil)
+    features = self.getAllFeatures
+    if file
+      open(file,'w') do |f|
+        features.each do |k,v|
+          f.puts k + ' ' + v.join(' ')
+        end
+      end
+    else
+      features.each do |k,v|
+        puts k + ' ' + v.join(' ')
+      end
+    end      
+    nil
+  end
+
+  # Returns a string of the n examples with highest target feature
+  def getTopRanking(n = 0)
+    self.sort{ |(k1,v1),(k2,v2)| v1[0]<=>v2[0]}.
+      reverse[0..(n-1)].map{|i| "#{i.first}\t#{i[1].first}"}.
+      join("\n")
+  end
+
+  def featname(index)
+    i = 0
+    s = ''
+    @config['Features'].each do |feature|
+      if i + @config[feature]['Dimensions'] > index
+        if s==''
+          s = 
+            if @config[feature]['Dimensions']==1
+              feature
+            else
+              "#{feature}_#{index - i}"
+            end
+        end
+      else
+        i += @config[feature]['Dimensions']
+      end
+    end
+    s
+  end
+
+  def to_s
+    (0...@config.dim).map do |i|
+      self.featname(i)
+    end.join(' ') + "\n" +
+    self.keys.sort.map do |key|
+      "#{key} #{self[key].join(' ')}"
+    end.join("\n")
+  end
+
+  # --- [] ---
+  # If indexing with a regular expression, a new SVMPrediction object is created
+  # containing all elements with matching keys.
+  def [](expr)
+    if expr.is_a? Regexp
+      subs = SVMPrediction.new
+      self.find_all { |(k,v)| k =~ expr }.each do |i|
+        subs[i[0]] = i[1]
+      end
+      subs
+    else
+      super(expr)
+    end
+  end
+
+end

data/lib/svmfeature2.rb

@@ -0,0 +1,98 @@
+class SVMFeature < Hash
+
+  attr_reader :cfg
+
+  # config is either a file object or a string object.
+  def initialize(config)
+    @cfg = if config.is_a? SVMFeaturesConfig
+             config
+           else
+             SVMFeaturesConfig.new(config)
+           end
+    updateFromDatabases!
+  end
+
+  def [](key)
+    return super.join(' ') if super
+    "I don't have it!"
+  end
+
+  #def []=(key,val)
+  #  STDERR.puts "You can't just PUT a value in SVMFeature..."
+  #  nil
+  #end
+
+  def updateFromDatabases!
+    @features = @cfg['Features'].map do |feature|
+               readDatabase(feature)
+             end
+    sizes = @features.map{|h| h.size}
+    minsize = sizes.min
+    index = sizes.index{|i| i==minsize}
+    @features[index].each do |k,v|
+      unless @features.find{|h| !h[k] or h[k]=~/ERROR/}
+        self[k] = @features.map{|h| h[k]}#.join(' ')
+      end
+    end
+  end
+
+  def readDatabase(feature)
+    file = getDataFileToRead(feature)
+    return {} unless file
+    open(file,'r') do |f|
+      if f.flock(File::LOCK_SH) 
+        begin
+          YAML.load(f) 
+        ensure
+          f.flock(File::LOCK_UN)
+        end
+      end
+    end
+  end
+
+  # Returns a string giving the path to the file holding
+  # the feature for the current settings.
+  # Returns nil if current setting do not match any file.
+  def getDataFileToRead(feature)
+    dfile = nil
+    filemapfname = File.join(@cfg[feature]['HomeDir'], 'filemap.yml')
+    return nil if !File.file?(filemapfname)
+    open(filemapfname,'r') do |f|
+      if f.flock(File::LOCK_SH) 
+        begin
+          filemap = YAML.load(f)
+          if cfg = filemap.find {|c| c['Config'] == @cfg[feature]}
+            dfile = File.join(@cfg[feature]['HomeDir'], cfg['FeatureFile']) 
+          else
+            dfile = nil
+          end
+        ensure
+          f.flock(File::LOCK_UN)
+        end
+      end
+    end
+    dfile
+  end
+
+  def getDataFileToWrite(feature)
+    return file if file=getDataFileToRead(feature)
+    file = File.join(@cfg[feature]['HomeDir'], 'filemap.yml')
+    open(file,'w') {} if !File.file?(file)
+    open(file,'r+') do |f|
+      if f.flock(File::LOCK_EX) 
+        begin
+          filemap = (tmp=YAML.load(f)) ? tmp : []
+          newdatafile = feature + filemap.size.to_s + '.yml'
+          filemap.push({'FeatureFile' => newdatafile,
+                         'Config'     => @config[feature] })
+          f.rewind
+          YAML.dump(filemap,f)
+        ensure
+          f.flock(File::LOCK_UN)
+        end
+      end
+    end
+    file
+  end
+
+end

data/lib/svmlab-config.rb

@@ -0,0 +1,215 @@
+# SVMLabConfig
+# is created by giving a text string as argument.
+
+require 'yaml'
+
+def method?(arg)
+  begin
+    method(arg)
+    true
+  rescue
+    false
+  end
+end
+
+# ---SVMFeature configuration---
+#
+# SVMFeature is initiated by a configuration file in YAML format.
+# Required fields in configuration are:
+# (all paths can be given either absolute or relative to BaseDir)
+#
+# ---
+# Features:
+# - <targetfeature> 
+# - <feature1>
+# - <feature2>
+# ...
+# BaseDir: <base directory>
+# DataSet: <path of file giving the dataset>
+#           OR
+#          <prefix of a set of files giving the dataset>
+# Example: If Dataset is not given, this gives name(s) of examples
+#          to use.
+# Groups: <range (n1..n2) or (n1...n2) in example name to use for grouping>
+#          OR
+#         <file prefix relative to BaseDir for files giving groups>
+# Methods: <path of .rb file holding all feature calculation methods>
+# --------------------------------------------------------------
+# targetfeature:
+#   HomeDir: <home directory>
+#       If HomeDir is not given, it will be set to BaseDir/featurename
+#   Method: <the method calculating this feature>
+#       If Method is not given, an attempts is made to acquire
+#       the feature from the database. If it fails, ERROR is reported.
+#   Dimensions: <the number of dimensions in this feature>
+#       If Dimensions is not given, it will be assumed to be 1
+#       and only the first value for each example will be used
+#   <Further specific configuration of this feature>
+# feature1:
+#   HomeDir: <home directory>
+#   Method: <the method calculating this feature>
+#   Dimensions: <the number of dimensions in this feature>
+#   <Further specific configuration of this feature>
+# feature2:
+# ...
+# featureN:
+#    If featureN configuration is not given, then all default settings
+#    will apply to this feature.
+# ...
+#
+#
+class SVMFeaturesConfig < Hash
+
+  attr_reader :cfg, :dim
+
+  #----------------SVMFeatureConfig--------------------------------------------
+  class SVMFeatureConfig < Hash
+    def initialize(maincfg, feature)
+      if !(cfg = maincfg[feature])
+        raise ArgumentError, "#{feature} configuration not given."
+      else
+        cfg.each do |k,v|
+          self[k] = v
+        end
+      end
+      # Method
+      method = self['Method']
+      if !method
+        raise ArgumentError, "Method for #{feature} not given."
+      end
+      if !method? method
+        raise ArgumentError, "Method #{method} for #{feature} not found."
+      end
+      # HomeDir
+      dir = self['HomeDir'] ? self['HomeDir'] : feature.dup
+      dir << '/' if dir[-1..-1]!='/'
+      self['HomeDir'] = maincfg.setupDirInfo(dir)
+      Dir.mkdir(self['HomeDir']) if !File.directory?(self['HomeDir'])
+      # Dimensions
+      self['Dimensions'] = self['Dimensions'] ? self['Dimensions'] : 1
+      # Meta-feature
+      if self['Features']
+        self['BaseDir'] = maincfg['BaseDir']
+      end
+
+    end
+
+    def to_s
+      self.to_yaml
+    end
+  end
+  #----------------------------------------------------------------------------
+
+  def initialize(arg)
+    begin
+      cfg = YAML.load(arg)
+      cfg.each do |k,v|
+        self[k] = v
+      end
+      # Set up dataset file
+      if self['DataSet']
+        self['DataSet'] = setupDirInfo(self['DataSet'])
+        if !File.exists? self['DataSet']
+          raise ArgumentError, "Cannot find dataset file #{self['DataSet']}."
+        end
+      end
+      # Get feature methods
+      if methods = self['Methods'] 
+        if methods.is_a? Array
+          methods.each { |method| require setupDirInfo(method) }
+        elsif methods.is_a? String
+          require setupDirInfo(methods)
+        end
+      end
+      # Set up each feature's configuration
+      raise ArgumentError, "Features not given." if !self['Features']
+      @dim = 0
+      self['Features'].each do |feature|
+        self[feature] = SVMFeatureConfig.new(self,feature)
+        @dim += self[feature]['Dimensions']
+      end
+    rescue ArgumentError, LoadError
+      raise $!,$!,nil
+    end
+  end
+  
+  def setupDirInfo(path)
+    if path !~ /^\// 
+      base = self['BaseDir']
+      raise ArgumentError, "BaseDir not given." if !base
+      path.sub!(/#{path}/, base + '/' + path)
+      path.sub!(/\/\//,'/')
+    end
+    path 
+  end
+
+end
+
+class SVMLabConfig
+
+  def initialize(cfg)
+    @cfg = if cfg.split("\n").size==1
+             YAML.load(File.new(cfg))
+           else
+             YAML.load(cfg)
+           end
+  end
+
+  def [](a)
+    @cfg[a]
+  end
+
+  def scales=(scale)
+    @cfg['SVM']['Scale'].each{ |k,v|
+      (0...v.size).each { |i|
+        @cfg['SVM']['Scale'][k][i] = 
+          if scale.is_a? String
+            scale.dup 
+          else 
+            scale 
+          end
+      } }
+  end
+
+  def C=(arg)
+    @cfg['SVM']['C'] = arg
+  end
+
+  def e=(arg)
+    @cfg['SVM']['e'] = arg
+  end
+
+  def to_s
+    @cfg.to_yaml
+  end
+
+  def sayhi
+    puts "Hello! I'm a SVMLabConfig object!"
+  end
+  
+#  SMVLabConfigStructure = 
+#    '--- 
+#Feature: 
+#  Features: 
+#  - exA
+#  exA: 
+#    Method: exB
+#    Dimensions: exC
+#  Methods: exD
+#  BaseDir: exE
+#  Groups: ExF
+#  DataSet: exG
+#SVM: 
+#  PosClassFrom: Float
+#  PShalf: Integer
+#  C: Float
+#  ScaleMethod: 
+#    Name: String
+#    Nhalf: Integer
+#    StepMethod: String
+#  e: Float
+#  Scale: 
+#    exA: 
+#    - exH
+#'
+end