frprep 0.0.1.prealpha

data/lib/common/Graph.rb

@@ -0,0 +1,345 @@
+# GraphNode: describes one node in a graph.
+#
+# A node may have an arbitrary number of parents (sources of incoming edges)
+# and an arbitrary number of children (targets of outgoing edges)
+#
+# All edges are labeled and directed
+#
+# The add_parent, add_child, remove_parent, remove_child methods
+# take care of both ends of an edge 
+# (i.e. n1.add_child(n2, label) also adds n1 as parent of n2 with edge label 'label'
+#
+# It is possible to create a 'pointer' rather than an edge:
+#     n1.add_child(n2, label, pointer_insteadof_edge => true) 
+# will create an edge from n1 to n2 labeled 'label' that is
+# listed under the outgoing edges of n1, but not among
+# the incoming edges of n2
+# The same option is available for add_parent, remove_parent, remove_child.
+
+class GraphNode
+
+  def initialize(id)
+    @id = id
+    @children = Array.new
+    @parents = Array.new
+    @features = Hash.new
+  end
+
+  # for Marshalling: 
+  # Dump just IDs instead of actual nodes from Parents and Children lists.
+  # Otherwise the Marshaller will go crazy following
+  # all the links to objects mentioned.
+  # After loading: replace IDs by actual objects with a little help
+  # from the caller.
+
+  def _dump(depth)
+    @id.to_s +
+      "QQSEPVALUESQQ" +
+      Marshal.dump(@features) +
+      "QQSEPVALUESQQ" +
+      @children.map { |label_child| 
+        label_child[0] + "QQSEPQQ" + label_child[1].id()
+      }.join("QQPAIRQQ") +
+      "QQSEPVALUESQQ" +
+      @parents.map { |label_parent|
+        label_parent[0] + "QQSEPQQ" + label_parent[1].id()
+    }.join("QQPAIRQQ")      
+  end
+
+  def GraphNode._load(string)
+    id, features_s, children_s, parents_s =
+      string.split("QQSEPVALUESQQ")
+
+    result = GraphNode.new(id)
+    result.fill_from_pickle(string)
+    return result
+  end
+
+  def fill_from_pickle(string)
+    id, features_s, children_s, parents_s =
+      string.split("QQSEPVALUESQQ")
+
+    @features = Marshal.load(features_s)
+
+    if children_s.nil? or children_s.empty?
+      @children = []
+    else
+      @children = children_s.split("QQPAIRQQ").map { |pair|
+	pair.split("QQSEPQQ")
+      }
+    end
+
+    if parents_s.nil? or parents_s.empty?
+      @parents = []
+    else
+      @parents = parents_s.split("QQPAIRQQ").map { |pair|
+	pair.split("QQSEPQQ")
+      }
+    end
+  end
+
+  def recover_from_dump(node_by_id)
+    @children = @children.map { |label_id| [label_id[0], node_by_id.call(label_id[1])] }
+    @parents = @parents.map { |label_id| [label_id[0], node_by_id.call(label_id[1])] }
+  end
+
+  # ID-related things
+
+  def ==(other_node)
+    unless other_node.kind_of? GraphNode
+      return false
+    end
+    @id == other_node.id()
+  end
+
+  def id()
+    return @id
+  end
+
+  def chid(newid)
+    @id = newid
+  end
+
+  # setting and retrieving features
+
+  def get_f(feature)
+    return @features[feature]
+  end
+
+  def set_f(feature, value)
+    @features[feature] = value
+  end
+
+  def add_f(feature, value)
+    unless @features[feature].nil?
+      raise "Feature " + feature + "already set."
+    end
+    set_f(feature, value)
+  end
+  
+  # ancestors 
+
+  def parents()
+    return @parents.map { |label_parent| 
+      label_parent[1] }
+  end
+  
+  def parent_labels()
+    return @parents.map { |label_parent| label_parent[0] }
+  end
+
+  def parent_label(parent)
+    @parents.each { |label_parent|
+      if label_parent[1] == parent 
+	return label_parent[0]
+      end
+    }
+    return nil
+  end
+
+  def parents_with_edgelabel()
+    return @parents
+  end
+
+  def each_parent()
+    @parents.each { |label_parent| yield label_parent[1] }
+  end
+
+  def each_parent_with_edgelabel()
+    @parents.each { |label_parent| yield label_parent}
+  end
+
+  def parents_by_edgelabels(labels)
+    return @parents.select { |label_parent|
+      labels.include? label_parent[0]
+    }.map { |label_parent|
+      label_parent[1]
+    }
+  end
+
+  def add_parent(parent, edgelabel, varhash={})
+    @parents << [edgelabel, parent]
+
+    # and vice versa: add self as child to parent
+    unless varhash["pointer_insteadof_edge"]
+      unless parent.children_with_edgelabel().include? [edgelabel, self]
+        parent.add_child(self, edgelabel)
+      end
+    end
+  end
+
+  def remove_parent(parent, edgelabel, varhash={})
+    @parents = @parents.reject { |label_child| 
+      label_child.first == edgelabel and
+	label_child.last == parent
+    }
+
+    # and vice versa: remove self as child from parent
+    unless varhash["pointer_insteadof_edge"]
+      if parent.children_with_edgelabel().include? [edgelabel, self]
+        parent.remove_child(self, edgelabel)
+      end
+    end
+  end
+
+  def indeg()
+    return @parents.length()
+  end
+
+  def ancestors
+    return ancestors_noduplicates([], [])
+  end
+
+  def ancestors_by_edgelabels(labels)
+    return ancestors_noduplicates([], labels)
+  end
+
+  # descendants
+
+  def children()
+    return @children.map { |label_child| label_child[1] }
+  end
+
+  def child_labels()
+    return @children.map { |label_child| label_child[0] }
+  end
+
+  def child_label(child)
+    @children.each { |label_child|
+      if label_child[1] == child
+	return label_child[0]
+      end
+    }
+    return nil
+  end
+
+  def children_with_edgelabel()
+    return @children
+  end
+
+  def each_child()
+    @children.each { |label_child| yield label_child[1]}
+  end
+
+  def each_child_with_edgelabel()
+    @children.each { |label_child| yield label_child } 
+  end
+
+  def children_by_edgelabels(labels)
+    return @children.select { |label_child|
+      labels.include? label_child[0] 
+    }.map { |label_child|
+      label_child[1]
+    }
+  end
+
+  def add_child(child, edgelabel, varhash={})
+    @children << [edgelabel, child]
+
+    # and vice versa: add self as parent to child
+    unless varhash["pointer_insteadof_edge"]
+      unless child.parents_with_edgelabel().include? [edgelabel, self]
+        child.add_parent(self, edgelabel)
+      end
+    end
+  end
+
+  def remove_child(child, edgelabel, varhash={})
+    @children = @children.reject { |label_child| 
+      label_child.first == edgelabel and
+	label_child.last == child
+    }
+
+    # and vice versa: remove self as parent from child
+    unless varhash["pointer_insteadof_edge"]
+      if child.parents_with_edgelabel().include? [edgelabel, self]
+        child.remove_parent(self, edgelabel)
+      end
+    end
+  end
+
+  def change_child_label(child, oldlabel, newlabel, varhash={})
+    if @children.include? [oldlabel, child]
+      remove_child(child,oldlabel, varhash)
+      add_child(child, newlabel, varhash)
+    end
+  end
+
+  def remove_all_children(varhash={})
+    each_child_with_edgelabel { |label, child|
+      remove_child(child, label, varhash)
+    }
+  end
+
+  def set_children(list, varhash={})
+    #### CAUTION: set_children must be called with an "internal format" list of parents:
+    ####          instead of using [node, edgelabel], use [edgelabel, node]
+    remove_all_children(varhash)
+
+    @children = list
+  end
+
+  def outdeg()
+    return @children.length()
+  end
+
+  def yield_nodes()
+    arr = Array.new
+    if outdeg() == 0
+      arr << self
+    end
+    each_child { |c| 
+      if c.outdeg() == 0
+	arr << c
+      else
+	arr.concat c.yield_nodes
+      end
+    }
+    return arr
+  end
+
+  def descendants
+    return descendants_noduplicates([], [])
+  end
+
+  def descendants_by_edgelabels(labels)
+    return descendants_noduplicates([], labels)
+  end
+
+  protected
+
+  def descendants_noduplicates(nodes, labels)
+    each_child_with_edgelabel() { |l_c|
+      if labels.empty? or labels.include? l_c[0]
+	unless nodes.include? l_c[1]
+	  nodes = l_c[1].descendants_noduplicates(nodes << l_c[1], labels)
+	end
+      end
+    }
+    return nodes
+  end
+  
+  def ancestors_noduplicates(nodes, labels)
+    each_parent_with_edgelabel() { |l_p|
+      if labels.empty? or labels.include? l_p[0]
+	unless nodes.include? l_p[1]
+	  nodes = l_p[1].ancestors_noduplicates(nodes << l_p[1], labels)
+	end
+      end
+    }
+    return nodes
+  end
+
+  #### CAUTION: set_parents must be called with an "internal format" list of parents:
+  ####          instead of using [node, edgelabel], use [edgelabel, node]
+
+  def set_parents(list, varhash={})
+    each_parent_with_edgelabel { |label, parent|
+      remove_parent(parent, label, varhash)
+    }
+
+    list.each { |label, parent|
+      add_parent(label, parent)
+    }
+  end
+end

data/lib/common/ISO-8859-1.rb

@@ -0,0 +1,24 @@
+# KE changed July 05: now no inclusion of modules required,
+# and names changed from REXML.Encodign to UtfIso
+
+module UtfIso
+  # Convert from UTF-8
+  def UtfIso.to_iso_8859_1(content)
+    array_utf8 = content.unpack('U*')
+    array_enc = []
+    array_utf8.each do |num|
+      if num <= 0xFF
+        array_enc << num
+      else
+        # Numeric entity (&#nnnn;); shard by  Stefan Scholl
+        #	   array_enc += to_iso_8859("&\##{num};").unpack('C*')
+      end
+    end
+    array_enc.pack('C*')
+  end
+
+  # Convert to UTF-8
+  def UtfIso.from_iso_8859_1(str)
+    str.unpack('C*').pack('U*')
+  end
+end

data/lib/common/ML.rb

@@ -0,0 +1,186 @@
+# sp 24 08 04
+
+# this file provides a very simple wrapper for using different ML systems
+# all you need to do is to write the appropriate learner class
+# and insert them in the initialize routine here in ML()
+#
+# available at the moment: 
+# * timbl (memory-based learner) 
+# * mallet-maxent (another maxent system)
+# * maxent (the OpenNLP maxent system)
+
+# part of contract: learner is not initialised unless it is either trained or read
+
+require "common/Optimise"
+
+class Classifier
+
+  @@learners = [
+    ["timbl", "Timbl", "Timbl"],
+#    ["mallet", "Mallet", "Mallet"],
+    ["maxent", "Maxent", "Maxent"]
+  ]
+
+  def initialize(learner,params)
+
+    @ready = false
+
+    if params[0] == "optimise"
+      params.shift
+      @optimise = true
+    else
+      @optimise = false      
+    end
+
+    program_path = ""
+    begin
+      program_path = params.shift.chomp
+      unless FileTest.exist? program_path
+        $stderr.puts "Error: Could not find classifier system at " + program_path
+        $stderr.puts "Perhaps an erroneous entry in your experiment file?"
+        exit 1
+      end
+    rescue NoMethodError
+      $stderr.puts "Error: No program path provided for classifier system."
+    end
+
+    # try to find our learner in the pre-set list of learners
+    learner_tuple = @@learners.assoc(learner)
+    unless learner_tuple
+      $stderr.puts "Error: I don't know the learner " + learner.to_s
+      $stderr.puts "Perhaps an erroneous entry in your experiment file?"
+      exit 1
+    end
+
+    learner_name, learner_filename, learner_classname = learner_tuple
+    require "common/#{learner_filename}"
+    @learner = eval(learner_classname).new(program_path,params)
+  end
+
+  # a classifier can (and has to be) either trained or read
+  def train(trainfile, classifier_file=nil)    
+    # train on the training data in trainfile
+    # make sure we produce a valid file name
+    
+    # it is possible to directly specify a filename for storing the classifier
+
+    trainfile.gsub!(/[<>]/,"")
+    trainfile.gsub!(/ /,"_")
+    if @optimise
+      STDERR.puts "[ML] using feature optimisation"
+      @optimiser = Optimise.new
+      @optimiser.init_from_data(trainfile)
+      optimisedfile = trainfile+".opted"
+      @optimiser.apply(trainfile,optimisedfile)
+      @learner.train(optimisedfile,classifier_file)
+      File.delete(optimisedfile)
+    else
+      STDERR.puts "[ML] no feature optimisation"
+      @learner.train(trainfile,classifier_file)      
+    end
+    @ready = true
+  end
+
+  
+  # returns true iff reading the classifier from the file has had success
+  
+  def read(classifier_file)
+    # make sure we produce a valid file name
+    classifier_file.gsub!(/[<>]/,"")
+    classifier_file.gsub!(/ /,"_")
+    
+    # read file, if present    
+    
+    status = @learner.read(classifier_file)
+    
+    # if reading has failed, return "false"
+    unless status
+      STDERR.puts "reading from #{classifier_file} did not succeed"
+      return status
+    end
+    
+    # read optimisation, if desired
+    if @optimise
+      optimisations_filename = Optimise.recommended_filename(classifier_file)
+      unless FileTest.exists? optimisations_filename
+        STDERR.puts "[ML] Error: attempted to read stored optimisation, but file does not exist"
+        return false
+      else
+	@optimiser = Optimise.new
+        @optimiser.init_from_file(optimisations_filename)
+      end
+    end
+    
+    @ready = true
+    return true
+    
+  end
+
+  # a classifier can be stored somewhere. This can be more than one file (classifier-specific), 
+  # but all files start with "classifier_file"
+
+  def write(classifier_file)
+    # make sure we produce a valid file name
+    classifier_file.gsub!(/[<>]/,"")
+    classifier_file.gsub!(/ /,"_")
+    @learner.write(classifier_file)
+    if @optimise
+      @optimiser.store(Optimise.recommended_filename(classifier_file))
+    end
+  end
+
+  ###
+  # exists?
+  # check if a classifier is living at some particular path
+
+  def exists?(classifier_file)
+    classifier_file.gsub!(/[<>]/,"")
+    classifier_file.gsub!(/ /,"_")
+    return @learner.exists?(classifier_file)
+  end
+  
+  # a classifier can be applied 
+
+  # returns true iff application has had success
+  
+  def apply(testfile,outfile) # test either on the training or the test data in the specified dir
+    # make sure we produce a valid file name
+    testfile.gsub!(/[<>]/,"")
+    testfile.gsub!(/ /,"_")
+    # make sure we produce a valid file name
+    outfile.gsub!(/[<>]/,"")
+    outfile.gsub!(/ /,"_")
+    
+    unless @ready
+      STDERR.puts "[ML] Warning: learner not ready for testing! Must be trained or read."
+      return false
+    end    
+    
+    # do we have a testfile?
+    
+    unless FileTest.exists?(testfile)
+      STDERR.puts "[ML] Warning: could not find testfile (maybe empty test set?)."
+      return false
+    end
+    
+    if @optimise      
+      optimisedfile = testfile+".opted"
+      @optimiser.apply(testfile,optimisedfile)
+      return @learner.apply(optimisedfile,outfile)
+      File.delete(optimisedfile)
+    else
+      return @learner.apply(testfile,outfile)
+    end
+    
+  end
+
+  ###
+  # read classifier result file,
+  # returns a list of instance_results
+  # where an instance_result is a list of pairs [label, confidence]
+  # where the pairs are sorted by confidence
+  def read_resultfile(file)
+    return @learner.read_resultfile(file)
+  end
+
+end