svmlab 1.0.0

data/lib/svmlab-irb.rb

@@ -0,0 +1,98 @@
+require 'irb/completion'
+require 'rubygems'
+require 'svmlab'
+
+#require 'wirble'
+#Wirble.init
+#Wirble.colorize
+
+#IRB.conf[:PROMPT][:SVMLab] = {       # name of prompt mode
+#  :PROMPT_I => "SVMLab:%03n:%i> ",   # normal prompt
+#  :PROMPT_S => "SVMLab:%03n:%i%l ",  # prompt for continuing strings
+#  :PROMPT_C => "SVMLab:%03n:%i* ",   # prompt for continuing statement
+#  :RETURN => "    ==>%s\n"           # format to return value
+#}
+ANSI_RESET      = "\033[0m"
+ANSI_GREEN     = "\033[2;32m"
+ANSI_BLUE     = "\033[2;34m"
+
+IRB.conf[:PROMPT][:SVMLab] = {       # name of prompt mode
+  #:PROMPT_I => "#{ANSI_BLUE}SVMLab>> #{ANSI_RESET}", # normal prompt
+  :PROMPT_I => "SVMLab > ",   # normal prompt
+  :PROMPT_S => "        %l ",  # prompt for continuing strings
+  :PROMPT_C => "       > ",   # prompt for continuing statement
+  #:RETURN => "#{ANSI_GREEN}     ==> %s\n#{ANSI_RESET}"  # format to return value
+  :RETURN => "     ==> %s\n"           # format to return value
+}
+
+IRB.conf[:PROMPT_MODE] = :SVMLab
+
+
+# Sebastian Delmont
+# Pretty print methods
+ANSI_BOLD       = "\033[1m"
+#ANSI_RESET      = "\033[0m"
+ANSI_LGRAY    = "\033[0;37m"
+ANSI_GRAY     = "\033[1;30m"
+
+def pm(obj, *options) # Print methods
+  methods = obj.methods
+  methods -= Object.methods unless options.include? :more
+  filter = options.select {|opt| opt.kind_of? Regexp}.first
+  methods = methods.select {|name| name =~ filter} if filter
+
+  data = methods.sort.collect do |name|
+    method = obj.method(name)
+    if method.arity == 0
+      args = "()"
+    elsif method.arity > 0
+      n = method.arity
+      args = "(#{(1..n).collect {|i| "arg#{i}"}.join(", ")})"
+    elsif method.arity < 0
+      n = -method.arity
+      args = "(#{(1..n).collect {|i| "arg#{i}"}.join(", ")}, ...)"
+    end
+    klass = $1 if method.inspect =~ /Method: (.*?)#/
+    [name, args, klass]
+  end
+  max_name = data.collect {|item| item[0].size}.max
+  max_args = data.collect {|item| item[1].size}.max
+  data.each do |item| 
+    print " #{ANSI_BOLD}#{item[0].rjust(max_name)}#{ANSI_RESET}"
+    print "#{ANSI_GRAY}#{item[1].ljust(max_args)}#{ANSI_RESET}"
+    print "   #{ANSI_LGRAY}#{item[2]}#{ANSI_RESET}\n"
+  end
+  data.size
+end
+
+# To enable history saving between sessions
+IRB.conf[:SAVE_HISTORY] = 100
+
+# Stian Haklev / Joel VanderWerf
+# To reduce lengthy output
+class IRB::Context
+  attr_accessor :max_output_size
+
+  alias initialize_before_max_output_size initialize
+  def initialize(*args)
+    initialize_before_max_output_size(*args)
+    @max_output_size = IRB.conf[:MAX_OUTPUT_SIZE] || 70
+  end
+end
+
+class IRB::Irb
+  def output_value
+    text =
+      if @context.inspect?
+        sprintf @context.return_format, @context.last_value.inspect
+      else
+        sprintf @context.return_format, @context.last_value
+      end
+    max = @context.max_output_size
+    if text.size < max
+      puts text
+    else
+      puts text[0..max-1] + "..." + text[-2..-1]
+    end
+  end
+end 

data/lib/svmlab-optim.rb

@@ -0,0 +1,556 @@
+
+#########################################################################################
+#   OPTIMIZATION METHODS
+#########################################################################################
+
+
+class SVMLab
+
+  private
+
+  # --- updateConfig! ---
+  # This is the interface between an array of variables
+  # to be changed and a configuration hash.
+  # It is to be used by the pattern search, but this
+  # function knows nothing about pattern search. It is 
+  # merely an interface between the hash and the 
+  # array describing a point in the search space.
+  # cfg: A hash containing what should be updated.
+  # update: An array of paths to what should be updated
+  #         and the new value.
+  # output: The cfg hash with updated values.
+  #
+  # EXAMPLE (yaml representation):
+  #   cfg =
+  #     --- 
+  #     a: 
+  #       a: 0
+  #       b: 1
+  #     b: 2
+  #   update =
+  #     --- 
+  #     - "a : b : 7 : exp1"
+  #     - "b : 8 : exp1"
+  def updateConfig!(cfg, steps)
+    # For each variable to be updated
+    steps.each do |upd|
+      # Split the path into its levels
+      path = upd.split(/\s\:\s/)
+      # Check if the current value should be an integer
+      path[-2] = if (p=path[-2]) =~ /^\d+$/ then p.to_i
+                 elsif p =~ /^\d+\.\d+$/ then p.to_f
+                 else p end
+      # Go through the path and update the hash's value
+      (0...path.size-2).inject(cfg) { |c,k| 
+        # If this key matches an integer, assume it's an integer
+        if path[k] =~ /^\d+$/ then path[k] = path[k].to_i end
+        if k == path.size-3
+          c[path[k]] = path[-2]
+        else
+          c[path[k]]
+        end
+      }
+    end
+    cfg
+  end
+  
+  def scrambleArray(arr)
+    b = []
+    a = arr.clone
+    while a.size > 0
+      b << a.delete_at(rand(a.size))
+    end
+    b
+  end
+  
+  # --- getPSsteps ---
+  def getPSsteps(cfg)
+    array = []
+    #1. Walk through cfg[Feature][Features] and look for scalings
+    cfg['Feature']['Features'].each do |feature|
+      if sc = cfg['SVM']['Scale'][feature]
+        sc.inject(0) do |i,sci|
+          if sci.is_a? String
+            sciarr = sci.split
+            if sciarr.size==3 and sciarr[1] =~ /step/
+              array.push('SVM : Scale : '+feature+' : '+i.to_s+' : '+sciarr[0]+' : '+sciarr[2])
+            end
+          end
+          i+=1
+        end
+      end
+    end
+    array = scrambleArray(array)
+    #2. Check SVM parameters
+    if c = cfg['SVM']['C'] and c.is_a? String
+      if (carr = c.split).size ==3 and carr[1] =~ /step/
+        array.push('SVM : C : '+carr[0]+' : '+carr[2])
+      end
+    end
+    if c = cfg['SVM']['e'] and c.is_a? String
+      if (carr = c.split).size ==3 and carr[1] =~ /step/
+        array.push('SVM : e : '+carr[0]+' : '+carr[2])
+      end
+    end
+    if c = cfg['SVM']['g'] and c.is_a? String
+      if (carr = c.split).size ==3 and carr[1] =~ /step/
+        array.push('SVM : g : '+carr[0]+' : '+carr[2])
+      end
+    end
+    if array.size>0 then array else nil end
+  end
+
+  # --- makeUpdate ---
+  # Creates the update array required by 'updateConfig!'.
+  # steps : Array of strings.
+  #         Each string :
+  #         <lvl 1> : <lvl 2> : ... : <lvl n> : <val> : <X / expX>
+  # movearr : Array of step sizes corresponding to the steps array.
+  #           E.g. movearr = [1, 0] means change the first element
+  #           of the steps array as curval + 1 * stepsize and leave
+  #           the second element unchanged.
+  # Returns a steps array.
+  def makeUpdate(steps,movearr = nil)
+    movearr = Array.new(step.size,0) if !movearr
+    steps.zip(movearr).map { |pm|
+      tmparr = pm[0].split(/\s\:\s/)
+      tmparr[0...-2].join(' : ') + ' : ' + # 1. All leading fields
+      if (num = if tmparr[-1] =~ /^exp\d/  # 2. The value
+                  (tmparr[-2].to_f * 10 ** (tmparr[-1].sub(/exp/,'').to_f * pm[1])).to_s
+                elsif tmparr[-1] =~ /^\d$/
+                  (tmparr[-2].to_f + tmparr[-1].to_f * pm[1]).to_s
+                end) =~ /\.0$/ 
+      then num.to_i.to_s 
+      else num end + ' : ' + 
+      tmparr.last                          # 3. The step size
+    }
+  end
+  
+    
+  # --- patternsearch ---
+  # Should possible be named 'coordinatesearch' since that is
+  # the algorithm that is implemented.
+  def patternsearch(cfg, x, verbose = nil)
+    if verbose
+      line = '#'+Array.new(60,'-').join
+      halfline = '#'+Array.new(43,'-').join
+      puts line,'#PATTERN SEARCH INITIAL POINT',{'init'=> x}.to_yaml,line
+    end
+    xn = x.size
+    s = Array.new(xn,0)
+    updateConfig!(cfg, makeUpdate(x,s))
+    delta = 1.0
+    iteration = 0
+    pred = SVMLab.new(cfg).crossvalidate
+    bestpred = pred
+    m = pred.rmsd
+    @pslog = [ {'time'  => Time.now,
+                'rmsd'  => m,
+                'cc'    => bestpred.cc,
+                'delta' => delta,
+                'x'     => x.dup,
+                'cfg'   => deepcopy(cfg)} ]
+    outputtitle =
+      (['Iter','Delta'] +
+       x.map{|xi|
+         arr = xi.split(' : ')
+         arr[arr.size>4 ? -4 : -3]} +
+       ['RMSD','CC']).join("\t")
+    outputline =
+      ([iteration, '-'] + 
+       x.map{|xi| "%.2f"%(xi.split(':')[-2].to_f)} + 
+       ["%.2f"%m, "%.2f"%bestpred.cc]).join("\t")
+    puts "PATTERN SEARCH"
+    puts outputtitle,outputline
+    if verbose
+      puts "#INITIAL VALUE : %.3f"%m + " (cc %.3f)"%pred.cc, line
+      puts halfline, "#\t\t\tDELTA -> " + delta.to_s, halfline
+      puts ( {'errors' => [m]}.to_yaml[5..-1] )
+    end
+    nhalf = 0
+    while nhalf <= cfg['SVM']['Optimization']['Nhalf'] do
+      s = Array.new(xn,0)
+      (0...xn).each do |i|
+        ei = Array.new(xn,0); ei[i] = 1
+        [-1,1].each do |d|
+          si = s.zip(ei).map { |se| se[0] + delta * d * se[1] }
+          updateConfig!(cfg, makeUpdate(x,si))
+          pred = SVMLab.new(cfg.to_yaml).crossvalidate 
+          val = pred.rmsd
+          #val = 1 - pred.cc # Try CC as measure instead
+          if m - m*10**-5 > val
+            s = si.dup
+            m = val
+            bestpred = pred
+            puts '#' + x[i].gsub("\s",'').split(/\:/)[0...-1].join(':') + 
+              ' -> ' + si[i].to_s +
+              "\tRMSD -> %.3f"%val +
+              "\t(cc %.3f)"%pred.cc if verbose
+            break
+          end
+        end
+      end
+      olddelta = delta
+      if s.detect{|ss| ss != 0}
+        updateConfig!(cfg, x = makeUpdate(x,s) )
+      else
+        delta = delta / 2
+        nhalf += 1
+        if verbose
+          puts "#\t\t\tDELTA -> " + delta.to_s
+          puts "- #{m}" if nhalf <= cfg['SVM']['Optimization']['Nhalf']
+        end
+        updateConfig!(cfg, x)
+      end
+      iteration += 1
+      outputline =
+        ([iteration, olddelta] + 
+         x.map{|xi| "%.2f"%(xi.split(':')[-2].to_f)} + 
+         ["%.2f"%bestpred.rmsd, "%.2f"%bestpred.cc]).join("\t")
+      puts outputline
+      puts halfline if verbose
+      @pslog.push( { 'time'  => Time.now,
+                     'rmsd'  => bestpred.rmsd,
+                     'cc'    => bestpred.cc,
+                     'delta' => delta,
+                     'x'     => x.dup,
+                     'cfg'   => deepcopy(cfg) } )
+    end
+    if verbose
+      puts line,'#'+Time.now.to_s + ' PATTERN SEARCH FINAL ERROR ' + m.to_s
+      puts ( {'finalerr' => m}.to_yaml[5..-1] )
+      puts line,'#PATTERN SEARCH FINAL POINT',{'final'=>x}.to_yaml[5..-1],line
+      puts ( {'endtime' => Time.now.to_s}.to_yaml[5..-1] )
+    end
+  end
+ 
+
+  # --- quasiNewton ---
+  def quasiNewton(cfg, x, verbose = nil)
+    delta = 1.0
+    nhalved = 0
+    s = Array.new(x.size,0) 
+    loop do
+      # Calculate the center value
+      updateConfig!(cfg, x = makeUpdate(x,s))
+      centerval =  SVMLab.new(cfg.to_yaml).crossvalidate.rmsd
+      # Calculate values around the current point
+      puts '***',"%.3f"%centerval,'***' if verbose
+      val = (0...x.size).map { |i|
+        ei=Array.new(x.size,0); ei[i] = 1
+        [-1,1].map { |d|
+          si = ei.map { |se| delta * d * se }
+          updateConfig!(cfg, makeUpdate(x,si))
+          v = SVMLab.new(cfg.to_yaml).crossvalidate.rmsd
+          #puts "If #{x[i]} is #{si[i]} then rmsd = #{v}"
+          v
+        } }
+     
+      s = Array.new(x.size,0)
+
+      case cfg['SVM']['Optimization']['StepMethod']
+
+      when 'Gradient'
+        # Calculate gradient and gradient norm
+        gradient = val.map{ |valx|
+          mini = valx[0]<valx[1] ? 0 : 1
+          if valx[mini] < centerval
+            (centerval - valx[mini]) * (mini==0 ? 1 : -1) 
+          else 0.0 end }
+        gradnorm = Math.sqrt(gradient.inject(0) { |sqsum,gi| sqsum += gi**2 })
+        delta = delta / 2 if gradnorm < 0.01
+        normedgrad = gradient.map{|gi| gi / gradnorm}
+        s = normedgrad.map{|gi| -gi * delta} # Negative gradient direction
+        if verbose
+          puts "Grad: |#{"%.3f"%gradnorm}| : #{gradient.map{|g| "%.2f"%g}.join(' ')}"
+        end
+
+      when 'MaxDescentAxis'
+        # Calculate the axis and direction that gives max descent
+        daxis = (0...val.size).zip(val).sort{|a,b| a[1]<=>b[1]}.first[0]
+        ddir = val[daxis][0] < val[daxis][1] ? 0 : 1
+        if val[daxis][ddir] < centerval 
+          s[daxis] = (ddir==0?-1:1) * delta
+          if verbose then puts "Changing #{daxis} to #{s[daxis]} " + 
+            "since it gives #{"%.3f"%val[daxis][ddir]}\n  #{x[daxis]}"
+          end
+        else 
+          delta = delta / 2
+          nhalved += 1
+          puts "Halving step size to #{delta}" if verbose
+        end
+        break if nhalved > cfg['SVM']['Optimization']['Nhalf']
+      end
+      #if verbose
+      #  puts 'S: ' + s.map{|si| si.is_a? Fixnum ? si.to_s : "%.2f"%si}.join(' ')
+      #end
+    end
+    updateConfig!(cfg, x)
+  end
+
+  def checkOptimization(cfg)
+    if x = getPSsteps(cfg)
+      #puts x
+      if cfg['SVM']['Optimization'] and 
+        (method = cfg['SVM']['Optimization']['Method'])
+        #puts '---',method,'---'
+        eval("#{method}(cfg, x, nil)")
+        #   Currently implemented methods:
+        #   patternsearch(cfg, x)
+        #   quasiNewton(cfg, x)
+      else
+        raise "Scale optimization requested in config but " +
+              "no optimization method given."
+      end 
+    else
+      cfg
+    end
+  end
+
+  def deepcopy(obj)
+    Marshal::load(Marshal::dump(obj))
+  end
+
+#  # ----------------------Old code from here----------------------------
+#
+#
+#  def patternsearch_fredrik(steps, psreport = '/tmp/ps', errormeasure = 'RMSD')
+#    stephalved = 0
+#    bestcfg = nil
+#    nstep = 0
+#    linesep = '**************'
+#    open(psreport,'w') { |f| f.puts linesep,'PATTERN SEARCH',Time.now,linesep }
+#    while stephalved<2 do 
+#      open(psreport,'a') { |f| f.puts linesep, "STEP #{nstep+=1}, " + Time.now.to_s, steps }
+#      pattern = getpattern(@cfg, steps)
+#      bestcfg = YAML.load(pattern[0]) if !bestcfg
+#      pindex = -1
+#      maxparallel = 7
+#      parallel = 0
+#      pattern.each do |p|
+#        rfile = "/tmp/patternsearch#{pindex+=1}.yml"
+#        open("/tmp/pattern#{pindex}.yml","w"){|f| f.puts p}
+#        if parallel >= maxparallel
+#          Process.wait
+#          parallel -= 1
+#        end
+#        fork do
+#          lab = SVMLab.new(p)
+#          err, predictions = lab.publish_crossvalidate(rfile)
+#        end
+#        parallel += 1
+#      end
+#      Process.waitall
+#      evaluation = {}
+#      (0...pattern.size).each do |pindex|
+#        rfile = File.new("/tmp/patternsearch#{pindex}.yml")
+#        evaluation[pindex] = YAML.load(rfile)['Evaluation'][errormeasure]
+#      end
+#      # Calculate gradient
+#      gradient = (1..(pattern.size-1)/2).map { |dim|
+#        evaluation[2*dim].to_f - evaluation[2*dim-1].to_f }
+#      gradnorm = Math.sqrt(gradient.inject(0) { |sqsum,dim| sqsum += dim**2 })
+#      open(psreport,'a') { |f| f.puts "Gradient" + 
+#        gradient.inject(''){|str,d| str+=" %.2f"%d} + " (%.2f)"%gradnorm }
+#      # Find minimum error configuration
+#      mincfg = evaluation.sort{|alfa,beta| alfa[1]<=>beta[1]}.first
+#      bestcfg = YAML.load( pattern[mincfg[0]] )
+#      open(psreport,'a') { |f| f.puts 'Error ' + evaluation[0].to_s }
+#      # Update steps
+#      if (mincfg[1] - evaluation[0]).abs < 1e-4
+#        open(psreport,'a') { |f| f.puts linesep,'HALVING' }
+#        steps.collect! do |step|
+#          if step=~/exp\d+/ 
+#            prefix = if step=~/int/ then 'intexp' else 'exp' end
+#            (step.split(/\s\:\s/)[0...-1] + [prefix+(step.split(/exp/).last.to_f/2).to_s]).join(' : ')
+#          else 
+#            (step.split(/\s\:\s/)[0...-1] + [(step.split(/\s\:\s/).last.to_f/2).to_s]).join(' : ')
+#          end
+#        end
+#        stephalved += 1
+#      else
+#        open(psreport,'a') { |f| f.puts linesep,'MOVING ' }
+#        steps.collect! do |step|
+#          path = step.split(/\s\:\s/)
+#          tmpcfg = bestcfg
+#          path[0...-3].each do |pathpart|
+#            tmpcfg = tmpcfg[pathpart]
+#          end
+#          (path[0...-2] + 
+#           [tmpcfg[ if path[-3]=~/^\d+$/ then path[-3].to_i 
+#                    else path[-3] end ].to_s ] + 
+#           [path.last]).join(' : ')
+#        end
+#        # Gradient descent
+#        #gradindex = -1
+#        #steps.collect! do |step|
+#        #  path = step.split(/\s\:\s/)
+#        #  tmpcfg = bestcfg
+#        #  path[0...-3].each do |pathpart|
+#        #    tmpcfg = tmpcfg[pathpart]
+#        #  end
+#        #  (path[0...-2] + 
+#        #   [ if path[-1] =~ /exp/
+#        #       path[-2].to_f * 10**(-path[-1].sub(/exp/,'').to_f * gradient[gradindex+=1] / gradnorm)
+#        #     else
+#        #       path[-2].to_f + (tmpcfg[path[-3]].to_f - path[-2].to_f).abs * gradient[gradindex+=1] / gradnorm
+#        #     end.to_s ] + 
+#        #   [path.last]).join(' : ')
+#        #end
+#      end
+#    end
+#    open(psreport,'a') { |f| f.puts linesep,Time.now,'FINAL STEPS',steps,linesep }
+#    bestcfg
+#  end
+#
+#  # --- getpattern ---
+#  # structure : A hash consisting the configuration. The leaves in the
+#  #             hash are ignored and only the structure is utilized.
+#  # steps : An array of strings in which each string describes the
+#  #         path in the "structure" hash to the parameter:
+#  #         <Level1> : <Level2> : ... : <Parameter> : <CurValue> : <Stepsize>
+#  #         "CurValue" is the value from which to create the pattern
+#  #         "Stepsize" is a string : 
+#  #           <x>, exp<x> or intexp<x> where x is a Float or Fixnum
+#  # returns : an array of strings in which each string is the YAML
+#  #           representation of the configuration. 
+#  #           The current point given by "CurValue" is returned as
+#  #           the first element in the array.
+#  def getpattern(structure, steps)
+#    pattern = []
+#    (['dontmove'] + steps).each do |step1|
+#      if step1=='dontmove' then [0] else [-1,1] end.each do |factor|
+#        steparr = []
+#        steps.each do |step2|
+#          s2 = step2.split(/\s\:\s/)
+#          if step1==step2 
+#            # This step will be changed according to factor and stepsize
+#            ss = if s2[-1] =~ /exp/
+#                   ss = s2[-2].to_f * 10**(s2[-1].split(/exp/).last.to_f * factor)
+#                   if s2[-1] =~ /int/ then ss.round else ss end
+#                 else
+#                   ss = s2[-2].to_f + s2[-1].to_f * factor
+#                   if ss.to_s =~ /\.0$/ then ss.to_i else ss end
+#                 end
+#            steparr.push( s2[0...-2].join(':') + ' ' + ss.to_s )
+#          else
+#            # Put the current point in steparr
+#            steparr.push( s2[0...-2].join(':') + ' ' + s2[-2] )
+#          end
+#        end
+#        # Generate config text for each step
+#        cfg = deepcopy(structure) # Do not change the argument given
+#        steparr.each do |step|
+#          intcfg = cfg
+#          path,val = step.split
+#          val = if val=~ /\.0$/ then val.to_i else val.to_f end
+#          path.split(/\:/)[0...-1].each do |p|
+#            intcfg = intcfg[p]
+#          end
+#          intcfg[ if (index = path.split(/\:/).last) =~ /^\d+$/ 
+#                  then index.to_i
+#                  else index      end ] = val
+#        end
+#        pattern.push( cfg.to_yaml )
+#      end
+#    end
+#    pattern
+#  end
+#
+#
+#  #########################################################################################
+#  # Grid search - possibly obsolete
+#
+#  # --- loo_svmoptim_gridsearch ---
+#  # Performs a grid search of the SVM hyperparameter space, i.e. does
+#  # a cross validation using different values of C, epsilon and gamma
+#  # (assuming we are doing epsilon regression with an RBF kernel)
+#  def loo_svmoptim_gridsearch()
+#    err, predictions = looCrossvalidate
+#    print 'Default: epsilon '+@par.eps.to_s
+#    print "\tgamma "+@par.gamma.to_s
+#    print "\tC "+@par.C.to_s
+#    print "\tError %.3f"%err
+#    puts
+#
+#    epsilon = -3..0
+#    gamma = -2..1
+#    cee = -1..2
+#    optimum = {}
+#    epsilon.each { |e|
+#      @par.eps = 10**e
+#      gamma.each { |g|
+#        @par.gamma = 10**g
+#        cee.each {|c|
+#          @par.C = 10**c
+#          err,predictions = looCrossvalidate
+#          print 'epsilon '+@par.eps.to_s
+#          print "\tgamma "+@par.gamma.to_s
+#          print "\tC "+@par.C.to_s
+#          print "\tError %.3f"%err
+#          if !optimum['err'] or optimum['err'] > err
+#            optimum['err'] = err
+#            optimum['epsilon'] = 10**e
+#            optimum['gamma'] = 10**g
+#            optimum['C'] = 10**c
+#          end
+#          print "\tOptimum %.3f"%optimum['err'] if optimum['err']
+#          puts
+#        }
+#      }
+#    }
+#    @par.eps = optimum['epsilon']
+#    @par.gamma = optimum['gamma']
+#    @par.C = optimum['C']
+#    optimum.each{ |key,val|
+#      puts key + ' = ' + val.to_s
+#    }
+#  end
+#
+#  # --- svmoptim_gridsearch ---
+#  # Performs a grid search of the SVM hyperparameter space, i.e. does
+#  # a cross validation using different values of C, epsilon and gamma
+#  # (assuming we are doing epsilon regression with an RBF kernel)
+#  def svmoptim_gridsearch(groups)
+#    err, predictions = crossvalidate(groups)
+#    print 'Default: epsilon '+@par.eps.to_s
+#    print "\tgamma "+@par.gamma.to_s
+#    print "\tC "+@par.C.to_s
+#    print "\tError %.3f"%err
+#    puts
+#
+#    epsilon = -3..0
+#    gamma = -2..1
+#    cee = -1..2
+#    optimum = {}
+#    epsilon.each { |e|
+#      @par.eps = 10**e
+#      gamma.each { |g|
+#        @par.gamma = 10**g
+#        cee.each {|c|
+#          @par.C = 10**c
+#          err,predictions = crossvalidate(groups)
+#          print 'epsilon '+@par.eps.to_s
+#          print "\tgamma "+@par.gamma.to_s
+#          print "\tC "+@par.C.to_s
+#          print "\tError %.3f"%err
+#          if !optimum['err'] or optimum['err'] > err
+#            optimum['err'] = err
+#            optimum['epsilon'] = 10**e
+#            optimum['gamma'] = 10**g
+#            optimum['C'] = 10**c
+#          end
+#          print "\tOptimum %.3f"%optimum['err'] if optimum['err']
+#          puts
+#        }
+#      }
+#    }
+#    @par.eps = optimum['epsilon']
+#    @par.gamma = optimum['gamma']
+#    @par.C = optimum['C']
+#    optimum.each{ |key,val|
+#      puts key + ' = ' + val.to_s
+#    }
+#  end
+
+end