complearn 0.6.2

data/ext/lib/CompLearnLib/RunEnv.rb

@@ -0,0 +1,150 @@
+module MRunEnv
+require 'CompLearnLib/CLConfig'
+
+# For now, use these paths
+class MPIRunner
+  @@MPIRUBY='mpi_ruby'
+
+  @@MPIRUN='mpirun'
+  @@MPILAMBOOT='lamboot'
+  @@MPIWIPE='wipe'
+  @@MPIHALT='lamhalt'
+
+end
+
+# Change this by setting OUTPUTDIR in your shell environment
+class DirNames
+  def DirNames.outputDir()
+    ENV['OUTPUTDIR'] || "/ufs/cilibrar/src/rubyquart/results"
+  end
+end
+
+class RunEnv
+  @@TMPDIR="/tmp"
+
+  @tmpCounter = 0
+  def RunEnv.getTemporaryFilename()
+    fname = format("%s/tmp-%05d.dat", @@TMPDIR, @tmpCounter)
+    @tmpCounter += 1
+    at_exit { File.exist?(fname) && File.unlink(fname) }
+    fname
+  end
+  def RunEnv.getTemporaryFilenames(howMany)
+    results = [ ]
+    howMany.times { results << RunEnv.getTemporaryFilename() }
+    results
+  end
+  def RunEnv.zrand(n)
+    rand(n)
+  end
+  def RunEnv.dirsInDir(dirname)
+    dirs = [ ]
+    for f in Dir.entries(dirname)
+      dirs << f if File.ftype(dirname+"/"+f)=='directory' && f != '.' && f != '..'
+    end
+    dirs.sort
+  end
+  def RunEnv.filesInDir(dirname)
+    files = [ ]
+    for f in Dir.entries(dirname)
+      files << f if File.ftype(dirname+"/"+f)=='file'
+    end
+    files.sort
+  end
+end
+
+class MPIRunner
+  def MPIRunner.getGoodHostsFile()
+    unless defined?(@hostFilename)
+      @hostFilename = SSHAgent.writeGoodHostFile
+    end
+    @hostFilename
+  end
+  def MPIRunner.rebootMPI()
+    MPIRunner.haltMPI
+    MPIRunner.bootMPI
+  end
+  def MPIRunner.bootMPI()
+    MPIRunner.getGoodHostsFile
+    result = `#{@@MPILAMBOOT} -x -v -s #{@hostFilename}`
+    puts result
+    @lamRunning = true
+  end
+  def MPIRunner.lamRunning?()
+    defined?(@lamRunning) && @lamRunning
+  end
+  def MPIRunner.haltMPI()
+    MPIRunner.getGoodHostsFile
+    #result = `#{@@MPIHALT}`
+    result = `#{@@MPIWIPE} -v #{@hostFilename}`
+    SSHAgent.runEverywhere('"killall mpi_ruby;killall lamboot;killall lamd"')
+    @lamRunning = false
+  end
+  def MPIRunner.stopMPIRubyScript()
+    #Process.kill(9, pid)
+    MPIRunner.haltMPI()
+  end
+  def MPIRunner.runMPIRubyScript(scriptArray) # returns PID of controller
+    scriptName = scriptArray[0]
+    scriptArgs = (scriptArray[1..-1]).join(' ')
+    puts "Running script #{scriptName} on #{`hostname`}"
+    MPIRunner.bootMPI unless lamRunning?
+#    res = fork()
+#    if (res)
+#      res
+#    else
+      exec("#{@@MPIRUN} C #{@@MPIRUBY} #{scriptName} #{scriptArgs}")
+#    end
+  end
+end
+
+class SSHAgent
+  def SSHAgent.isRunning?()
+    SSHAgent.checkHost(SSHAgent.primaryHost)
+  end
+  def SSHAgent.checkHost(hostname)
+    puts "Checking #{hostname}"
+    result = `unset DISPLAY ; setsid ssh #{hostname} hostname -s`
+    result.gsub!(/\s/,'')
+    hostname == result
+  end
+  def SSHAgent.primaryHost()
+    CLConfig.getDefaultConfig().hosts[0]
+  end
+  def SSHAgent.runAt(cmd, h)
+    if (h == 'dorado')
+      puts "Skipping running #{cmd} on dorado"
+    else
+      `unset DISPLAY ; setsid ssh #{h} #{cmd}`
+    end
+  end
+  def SSHAgent.runEverywhere(cmd)
+    [SSHAgent.primaryHost(), SSHAgent.helperHosts()].flatten.each { |h|
+      SSHAgent.runAt(cmd, h)
+    }
+  end
+  def SSHAgent.helperHosts()
+    CLConfig.getDefaultConfig().hosts[1..-1]
+  end
+  def SSHAgent.liveHosts()
+    unless SSHAgent.isRunning?
+      puts "Sorry, you forgot to start ssh-agent or run ssh-add"
+      exit(1)
+    end
+    alive = [ ]
+    SSHAgent.helperHosts.each { |h| alive << h if SSHAgent.checkHost(h) }
+    alive
+  end
+  def SSHAgent.writeGoodHostFile()  # returns filename of new hosts file
+    cfg = CLConfig.getDefaultConfig
+    fname = RunEnv.getTemporaryFilename()
+    f = File.open(fname, "wb")
+    cfg.hosts.each { |h|
+      f.write("#{h}\n")
+    }
+    f.close
+    fname
+  end
+end
+
+end

data/ext/lib/CompLearnLib/Task.rb

@@ -0,0 +1,39 @@
+
+require 'CompLearnLib/TaskMaster'
+
+module MTask
+
+class Task
+
+  attr_reader :tid, :sentTime
+
+  def aboutToSend()
+    @sentTime = time
+  end
+
+  def setTid(ntid)
+    @tid = ntid
+  end
+
+  def reply(result)
+    TaskMaster.reply(self,result)
+  end
+
+  def fetch(key)
+    TaskMaster.fetch(key)
+  end
+    
+  def execute()
+    reply(nil)
+  end
+
+end
+
+class Store
+  attr_reader :key, :val
+  def initialize(ukey, uval)
+    @key,@val = ukey,uval
+  end
+end
+
+end

data/ext/lib/CompLearnLib/TaskMaster.rb

@@ -0,0 +1,13 @@
+#
+# TaskMaster
+#
+# TaskMaster superclass
+
+require 'CompLearnLib/CLConfig'
+
+if CLConfig.getDefaultConfig().isSingleProcess? || !defined?(MPI)
+    require 'CompLearnLib/TaskMasterSingle'
+else
+    require 'CompLearnLib/TaskMasterMPI'
+end
+

data/ext/lib/CompLearnLib/TaskMasterMPI.rb

@@ -0,0 +1,112 @@
+
+unless defined?(MPI)
+  puts "Sorry, you seem to have started an MPIRuby script using"
+  puts "the wrong ruby interpretter.  Please try again something"
+  puts "like ruby parbegin.rb yourScriptName.rb"
+  exit 1
+end
+
+class ExitMsg
+end
+
+class TaskMaster
+
+  @@haveInitted = false
+
+  @cbs = { }
+  @free = [ ]
+  @slaves = [ ]
+  @tasks = { }
+  @maxTid = 0
+  @storage = { }
+
+  def TaskMaster.fetch(key)
+    @storage[key]
+  end
+
+  def TaskMaster.doMasterInit()
+    return if @@haveInitted
+    @@haveInitted = true
+    sz = MPI::Comm::WORLD.size()
+    puts "Starting calculation with #{sz} nodes"
+    Kernel.at_exit() {
+      exitmsg = ExitMsg.new
+      @slaves.each { |n|
+        MPI::Comm::WORLD.send(exitmsg, n, 0)
+      }
+    }
+    sz.times { |i| @free << i unless i == 0 }
+    @slaves = @free.clone
+  end
+
+  def TaskMaster.doSlaveLoop()
+    Process.setpriority(Process::PRIO_PROCESS, Process.pid, 19)
+    while true
+      task,status = MPI::Comm::WORLD.recv(0, 0)
+      if task.is_a?(ExitMsg)
+        exit(0)
+      end
+      if task.is_a?(Task)
+        task.execute()
+      end
+      if task.is_a?(Store)
+        @storage[task.key] = task.val
+      end
+    end
+  end
+
+  def TaskMaster.init()
+      if (MPI::Comm::WORLD.rank() == 0)
+        TaskMaster.doMasterInit()
+      else
+        TaskMaster.doSlaveLoop()
+      end
+  end
+
+  def TaskMaster.getFreeSlave()
+    waitForSlave() if @free.size == 0
+    nextSlave = @free.shift
+    nextSlave
+  end
+
+  def TaskMaster.waitForReply()
+    TaskMaster.waitForSlave()
+  end
+
+  def TaskMaster.reply(task, result)
+    MPI::Comm::WORLD.send([task.tid, result], 0, 0)
+  end
+
+  def TaskMaster.handleReply(reply, status)
+    tid,obj = reply
+    if @cbs[tid]
+      @cbs[tid].call(obj, @tasks[tid], status.source) 
+      @cbs[tid] = nil
+    end
+    @tasks[tid] = nil
+    @free << status.source
+  end
+
+  def TaskMaster.waitForSlave()
+      reply, status = MPI::Comm::WORLD.recv(MPI::Comm::ANY_SOURCE, 0)
+      TaskMaster.handleReply(reply,status)
+  end
+
+  def TaskMaster.storeEverywhere(key, val)
+    @slaves.each { |n|
+      MPI::Comm::WORLD.send(Store.new(key,val), n, 0)
+    }
+  end
+
+  def TaskMaster.enqueue(t, &cb)
+      @maxTid += 1
+      t.setTid(@maxTid)
+      @cbs[t.tid] = cb
+      @tasks[t.tid] = t
+      nextFreeSlave = TaskMaster.getFreeSlave
+      m = Marshal.dump(t)
+      MPI::Comm::WORLD.send(t, nextFreeSlave, 0)
+  end
+
+end
+

data/ext/lib/CompLearnLib/TaskMasterSingle.rb

@@ -0,0 +1,39 @@
+
+class TaskMaster
+
+  @lastReply = nil
+  @storage = { }
+
+  def TaskMaster.reply(t,reply)
+    @lastReply = reply
+  end
+
+  def TaskMaster.storeEverywhere(key, val)
+    @storage[key.clone] = val.clone
+  end
+
+  def TaskMaster.fetch(key)
+    @storage[key]
+  end
+
+  def TaskMaster.doMasterInit()
+  end
+
+  def TaskMaster.init()
+  end
+
+  def TaskMaster.waitForReply()
+    TaskMaster.waitForSlave()
+  end
+  def TaskMaster.waitForSlave()
+  end
+
+  def TaskMaster.handleReply(reply, status)
+  end
+
+  def TaskMaster.enqueue(t, &cb)
+      t.execute()
+      cb.call(@lastReply, t, 0)
+  end
+end
+

data/ext/lib/CompLearnLib/Tree.rb

@@ -0,0 +1,300 @@
+require 'CompLearnLib/Task'
+require 'CompLearnLib/RunEnv.rb'
+require 'CompLearnLib/CLConfig.rb'
+include MRunEnv
+include MTask
+
+module MTree
+
+class TreeTask < Task
+  def initialize(tree, tries, penalty)
+    @tree, @tries, @penalty = tree, tries, penalty
+  end
+  def execute()
+    dm = fetch('dm')
+    ts = TreeScore.makeFullList(dm)
+    ts.penalty = @penalty
+    best, bestscore = nil, nil
+    @tries.times {
+      maybe = @tree.clone
+      maybe.mutateComplex
+      score = ts.score(maybe)
+      if (best == nil || bestscore < score)
+        best = maybe
+        bestscore = score
+      end
+    }
+    reply([best, bestscore])
+  end
+end
+
+class Tree
+# consists of 2n - 2 nodes: n species followed by n-2 kernel nodes
+  attr_reader :edges
+  def initialize()
+    @edges = [ ]
+    @spm = nil
+  end
+  def clone()
+    t = Tree.new
+    t.copyEdges(self)
+    t
+  end
+  def <=>(other)
+    res = @edges.size - other.edges.size
+    return res unless res == 0
+    @edges.each_index { |i|
+      res = @edges[i].size - other.edges[i].size
+      return res unless res == 0
+      @edges[i].each_index { |j|
+        res = @edges[i][j] - other.edges[i][j]
+        return res unless res == 0
+      }
+    }
+    0
+  end
+  def copyEdges(t)
+    @edges = [ ]
+    t.edges.each_index { |i|
+      @edges[i] = t.edges[i].clone
+    }
+    @spm = nil
+  end
+  def makeName(i, names)
+    if (i < names.size)
+      return names[i].gsub(/[.].*/,'')
+    else
+      return CLConfig.getDefaultConfig.internalNodePrefix() + (i - names.size).to_s
+    end
+  end
+  def toDotString(names, title, desc)
+    result = "/* #{desc} */\ngraph #{title} {\n"
+    @edges.each_index { |i|
+      @edges[i].each { |j|
+      result += "#{makeName(i,names)} -- #{makeName(j,names)}\n"
+      }
+    }
+    result += "}\n"
+    result
+  end
+  def to_s()
+    result = ""
+    @edges.each_index { |i|
+      result += "#{i}->"+@edges[i].join(",")+"\n"
+    }
+    result
+  end
+  def Tree.randomTree(species)
+    raise "Not enough species" unless (species >= 3)
+    result = Tree.new
+    k = species - 2
+    nodes = species+k
+    (k-1).times { |i|
+      n = nil
+      begin
+        n = RunEnv.zrand(i+1) + species
+      end until result.getNeighborCount(n) <= 2
+      result.connect(i+species+1,n)
+    }
+    species.times { |i|
+      n = nil
+      begin
+        n = RunEnv.zrand(k) + species
+      end until result.getNeighborCount(n) <= 2
+      result.connect(i,n)
+    }
+    result
+  end
+  def calculateSPMFor(n)
+    @spm[n] = [ ]
+    curspm = @spm[n]
+    curspm[n] = n
+    visitted = { }
+    border = { n => true }
+    begin
+      border.each_key { |z|
+        visitted[z] = true
+        border.delete(z)
+        getNeighbors(z).each { |m|
+          unless visitted[m]
+            border[m] = true
+            curspm[m] = z
+          end
+        }
+      }
+    end until visitted.size == @edges.size
+  end
+  def calculateSPM()
+    @spm = [ ]
+    @edges.each_index { |n| calculateSPMFor(n) }
+  end
+  def findPath(a,b)
+    calculateSPM() unless @spm
+    path = [ ]
+    curspm = @spm[b]
+    cur = a
+    while (cur != b)
+      path << cur
+      cur = curspm[cur]
+    end
+    path << cur
+    path
+  end
+  def getNodeCount()
+    @edges.size
+  end
+  def getSpeciesCount()
+    (getNodeCount()+2) / 2
+  end
+  def getKernelCount()
+    getSpeciesCount() - 2
+  end
+  def isSpecies?(node)
+    getNeighborCount(node) == 1
+  end
+  def isKernel?(node)
+    getNeighborCount(node) == 3
+  end
+  def getNeighbors(node)
+    @edges[node]
+  end
+  def randomNeighbor(node)
+    @edges[node][RunEnv.zrand(@edges[node].size)]
+  end
+  def getNeighborCount(node)
+    return 0 unless @edges[node]
+    @edges[node].size
+  end
+  def connected?(a,b)
+    @edges[a] && @edges[a].include?(b)
+  end
+  def disconnect(a,b)
+    @edges[a].delete(b)
+    @edges[b].delete(a)
+    @edges[a] = nil if @edges[a].size == 0
+    @edges[b] = nil if @edges[b].size == 0
+    @spm = nil
+  end
+  def connect(a,b)
+    @edges[a] = [ ] unless @edges[a]
+    @edges[b] = [ ] unless @edges[b]
+    @edges[a] << b
+    @edges[b] << a
+    @edges[a].sort!
+    @edges[b].sort!
+    @spm = nil
+  end
+  def randomSpeciesNode()
+    n = nil
+    begin
+      n = randomNode()
+    end until isSpecies?(n)
+    n
+  end
+  def randomSpeciesNodes(count)
+    raise "Not enough species nodes" unless getSpeciesCount() >= count
+    res = { }
+    begin
+      res[randomSpeciesNode()] = true
+    end until res.size == count
+    res.keys
+  end
+  def randomNode()
+    RunEnv.zrand(getNodeCount())
+  end
+  def randomNodes(count)
+    raise "Not enough nodes" unless getNodeCount() >= count
+    res = { }
+    begin
+      res[randomNode()] = true
+    end until res.size == count
+    res.keys
+  end
+  def randomKernelNodes(count)
+    raise "Not enough kernel nodes" unless getKernelCount() >= count
+    res = { }
+    begin
+      res[randomKernelNode()] = true
+    end until res.size == count
+    res.keys
+  end
+  def mutateSpecies()
+    begin
+      a,b = randomSpeciesNodes(2)
+      na,nb = getNeighbors(a)[0],getNeighbors(b)[0]
+    end until na != nb
+    disconnect(a,na)
+    disconnect(b,nb)
+    connect(a,nb)
+    connect(b,na)
+  end
+
+  def mutateSubtreeInterchange()
+    p = nil
+    begin
+      a,b = randomKernelNodes(2)
+      p = findPath(a,b)
+    end until (p.size > 3)
+    na,nb = p[1],p[-2]
+    disconnect(a,na)
+    disconnect(b,nb)
+    connect(a,nb)
+    connect(b,na)
+  end
+
+  def mutateSubtreeTransfer()
+    begin
+      k1, k2 = randomNode, randomKernelNode()
+      p = findPath(k1,k2)
+    end until p.size > 2
+    i1 = p[1]
+    disconnect(k1,i1)
+    ms = getNeighbors(i1)
+    m1,m2,m3 = ms[0],ms[1],nil
+    begin
+      m3 = randomNeighbor(k2)
+    end until m3 != p[-2]
+    [[m1,i1], [m2,i1], [m3,k2]].each { |a| disconnect(a[0],a[1]) }
+    [[m1,m2], [k2,i1], [m3,i1], [k1,i1]].each { |a| connect(a[0],a[1]) }
+  end
+
+  def randomKernelNode()
+    n = nil
+    begin
+      n = randomNode()
+    end until isKernel?(n)
+    n
+  end
+
+  def mutateRandom()
+    begin
+    c = RunEnv.zrand(3)
+    end while c == 1 && getNodeCount <= 10
+    case c
+      when 0 then mutateSpecies
+      when 1 then mutateSubtreeInterchange
+      when 2 then mutateSubtreeTransfer
+    end
+  end
+
+  def mutateComplex()
+    orig = clone()
+    begin
+      mutateRandom
+    end while RunEnv.zrand(0) < 0.5 || (orig <=> self) == 0
+  end
+
+  def verifyTree()
+    oldNeighborCount = 0
+    @edges.each_index { |i|
+      s = @edges[i].size
+      return false if s != 1 && s != 3
+      oldNeighborCount = s if (s > oldNeighborCount)
+      return false if s < oldNeighborCount
+    }
+    return true
+  end
+  private :calculateSPM, :calculateSPMFor
+end
+
+end