shalmaneser 1.2.0.rc4 → 1.2.rc5

data/lib/common/Optimise.rb

@@ -1,195 +0,0 @@
-# sp 29 07 04
-# "optimise" c4.5 files by replacing all feature values which only
-# occur with one label by a new, common value.
-#
-# two modes of operation:
-# optimise <file>                -- optimise file and store optimisations in <file>.opts
-# optimise <file> <file.opts>    -- apply optimisation from file.opts to file
-
-class Optimise
-
-  def initialize
-    @ready = false
-  end
-
-  def init_from_data(infile) # find new optimisation
-    
-    STDERR.puts "[Optimise] computing new feature optimisation"
-
-    infile = File.new(infile)
-    labels = Array.new    
-    features = nil
-    @replacements = Array.new # for each feature, store the list of replacements
-    
-    # read data from infile into hash and initialise replacements array
-    while (line = infile.gets)
-      f_l = line.chomp.split(",")
-      
-      if features.nil? # first line: initialisation
-	features = Array.new # for each feature: array of feature values from file
-	f_l.each_index {|i|
-	  features[i] = Array.new
-	  @replacements[i] = Hash.new
-	}
-      end
-      labels << f_l.pop
-      f_l.each_index {|i|
-	features[i] << f_l[i]
-      }
-    end
-    infile.close
-
-    features.each_index {|findex| # traverse all features
-      
-      # for each feature *value*, find all label indices
-      
-      fvalues = features[findex]
-      
-      fval_to_label = Hash.new # record fval -> label mappings
-                                  # no label : nil
-                                  # one label: <label>
-                                  # two labels: false
-
-      fvalues.each_index {|inst_idx|
-	label = labels[inst_idx] # current label
-	fval = fvalues[inst_idx] # current feature value
-	seen_label = fval_to_label[fval] # previously seen label
-	if seen_label.nil?
-	  fval_to_label[fval] = label
-	elsif seen_label and seen_label != label
-	  fval_to_label[fval] = false
-	end
-      } # at the end, all fvals should be mapped to either <label> or false
-
-      # construct new feature value names
-      
-      new_fvals = Hash.new
-      labels.each {|label|
-	new_fvals[label] = "f"+findex.to_s+"_"+label.gsub(/\./,"")
-      }
-
-      # record all features values for which we have only seen one label in @replacements
-
-      fval_to_label.each_pair {|fval,label|
-	if fval == "[U]"
-	  puts "[U]: "+label.to_s+" "+new_fvals[label]
-	end
-	if label
-#	  STDERR.puts "replacement of "+fval+" by "+new_fvals[label]
-	  @replacements[findex][fval] = new_fvals[label]
-	end
-      }
-    
-    #   fvalues = features[findex]
-      
-#       l_to_v = Hash.new # label -> array of feature values
-#       v_to_l = Hash.new # feature value -> array of labels
-      
-#       fvalues.each_index {|inst| # traverse all instances 
-# 	fval = fvalues[inst]
-# 	label = labels[inst]
-	
-	
-# 	unless v_to_l.key?(fval) # add entry to v_to_l
-# 	  v_to_l[fval] = Array.new
-#           end
-# 	v_to_l[fval] << label
-	
-# 	unless l_to_v.key?(label) # add entry to l_to_v
-# 	  l_to_v[label] = Array.new
-# 	end
-# 	l_to_v[label] << fval          
-#       }
-        
-#       l_to_v.each_pair {|label,values|
-# 	newvalue = "f"+findex.to_s+"_"+label.gsub(/\./,"")
-# 	values.each {|value|
-# 	  if v_to_l[value].uniq.length == 1
-# 	    @replacements[findex][value] = newvalue
-# 	  end
-# 	}
-#       }
-     }
-    @ready = true
-  end
-  
-  def init_from_file(optsfile) # use old optimisation
-    optsinfile = File.new(optsfile)
-    @replacements = read(optsinfile) 
-    optsinfile.close
-    @ready = true
-  end
-  
-  def store(outfilename) # store data necessary to recreate optimisation
-    unless @ready
-      raise "[Optimise] Error: Cannot store un-initialised optimisation"
-    end
-    outfile = File.new(outfilename,"w") 
-    @replacements.each_index {|i| # for each feature
-      reps = @replacements[i]
-      outfile.puts "<"+i.to_s+">"
-      reps.each_pair{|old,new|
-	outfile.puts [old,new].join("\t")
-      }
-      outfile.puts "</"+i.to_s+">"
-    }
-    outfile.close
-  end
-
-  def apply(infilename,outfilename)    
-    unless @ready
-      raise "[Optimise] Error: Cannot apply un-initialised optimisation"
-    end
-
-    STDERR.puts "[Optimise] applying feature optimisation"
-    
-    infile = File.new(infilename)
-    outfile = File.new(outfilename,"w")
-    features = Array.new
-    labels = Array.new
-    
-
-    while (line = infile.gets)
-      tokens = line.chomp.split(",")
-      
-      unless tokens.length == @replacements.length
-	raise "[Optimise] Error: trying to optimise incompatible feature file!\nFile has "+features.length.to_s+" features, and we know replacements for "+@replacements.length.to_s+" features."
-      end
-      
-      label = tokens.pop
-      tokens.each_index {|f_idx|
-	fval = tokens[f_idx]
-	if @replacements[f_idx].key?(fval)
-	  tokens[f_idx] = @replacements[f_idx][fval]
-	end
-      }
-      tokens.push label
-      outfile.puts tokens.join(",")
-    end
-    outfile.close    
-  end
-  
-  private
-  
-  def read(infile)
-    @replacements = Array.new
-    while line = infile.gets
-      line.chomp!
-      if line =~ /<(\d+)>/
-	reps = Hash.new
-      elsif line =~ /<\/(\d+)>/
-	@replacements[$1.to_i] = reps
-      else
-	tokens = line.chomp.split("\t")
-	reps[tokens[0]] = tokens[1]
-      end
-    end
-    infile.close
-  end
-
-  # return recommended filename to store optimisation patterns for basefile
-  def Optimise.recommended_filename(basefile)
-    return basefile+".optimisations"
-  end
-  
-end

data/lib/common/Parser.rb

@@ -1,213 +0,0 @@
-# Alexander Koller 2003
-# extended Katrin Erk June 2003
-#
-# Classes that return a list of sentence DOMs, from various sources
-#
-# Each class in this file defines the following methods:
-#
-#   initialize(...)     "..." depends on the class
-#   extractDOMs()       return list of all s nodes as DOM objects
-#   each_s()            iterate over s nodes; may take less memory  
-
-
-require "rexml/document"
-
-class FileParser 
-
-  include REXML
-
-  def initialize(filename)
-    @file = File.new(filename)
-    @doc = nil
-  end
-
-  # returns an array of DOMs for the sentences
-  def extractDOMs()
-    ensureParsedDocument()
-    @doc.get_elements("/corpus/body/s")
-  end
-
-  # Iterates over all sentence nodes. This may be more memory
-  # efficient than using extractDOMs(), but isn't in this case.
-  def each_s()
-    extractDOMs().each { |dom| yield(dom) }
-  end
-
-  # Iterates over all sentence nodes. The block passed to this
-  # method should return a DOM object as a value. After the iteration
-  # has been completed, the contents of /corpus/body are then replaced
-  # by the list of these results.
-  # At the moment, this changes the FileParser object. This should
-  # probably change in the future, but I don't want to mess with
-  # cloning now.
-  def process_s!()
-    newBody = Element.new('body')
-    each_s { |dom| newBody.add_element( yield(dom) ) }
-
-    @doc.delete_element("/corpus/body")
-    @doc.elements["corpus"].add_element(newBody)
-
-    return @doc
-  end
-
-
-
-  private
-  
-  def ensureParsedDocument()
-    if @doc == nil then
-      @doc = Document.new(@file)
-    end
-  end
-
-      
-end
-
-
-
-
-#####################################################################
-
-
-
-
-class FilePartsParser
-  # <@file> = File object for the corpus
-  # <@head> = string up to the first <s> tag
-  # <@tail> = string after the last </s> tag
-  # <@rest> = string starting with the latest <s> tag (complete this to
-  # a <s>...</s> structure by reading up to next </s> tag)
-  # <@readCompletely> = boolean specifying whether there's still something
-  # left to read in the file
-
-  attr_reader :head, :tail
-
-  def initialize(filename)
-    @file = File.new(filename)
-    @readCompletely = false
-    # read stuff into @head and initialize @rest
-    @head = ''
-    begin
-      while true do
-	line = @file.readline() 
-	if line =~ /(.*)(<s\s.*)/ then
-	  @head = @head << $1
-	  @rest = $2
-	  break
-	elsif line =~ /^(.*)(<\/body[\s>].*)$/
-	  # empty corpus
-	  @head = @head << $1
-	  @tail = $2
-	  while (line = @file.readline())
-	    @tail << "\n" + line
-	  end
-	  @readCompletely = true
-	  break
-	else
-	  @head = @head << line
-	end
-      end
-    rescue EOFError
-      @readCompletely = true
-    end
-  end
-
-  def close()
-    @file.close()
-  end
-
-  def extractDOMs()
-    allDOMs = Array.new
-
-    process_s!() { |dom| 
-      allDOMs.push(dom) 
-      Element.new("x")
-    }
-    return allDOMs
-  end
-
-  def each_s()
-    process_s!() { |dom| 
-      yield(dom) 
-      Element.new("x")
-    }
-  end
-
-  # This function returns the string for the modified corpus.
-  # It doesn't change the internal state of the FilePartsParser,
-  # and is much more memory (and probably time) efficient than
-  # FileParser#process_s!.
-  # The block that is called by the method is given an element
-  # as its argument and is expected to return a changed element.
-  def process_s!()
-    if @readCompletely
-      return
-    end
-
-    ret = ''
-    scan_s() { |element|
-      # Process the <s> ... </s> element
-      doc = Document.new(element)
-      elt = doc.root
-      changedElt = yield(elt)
-
-      changedEltAsString = ''
-      changedElt.write(changedEltAsString, 0)
-      ret <<= changedEltAsString
-    }
-
-    return ret
-  end
-
-  # KE 12.6.03: scan_s : 
-  # doesn't parse a sentence before yielding it
-  # doesn't allow for any changes
-  # but otherwise the same as process_s!
-  def scan_s()
-    if @readCompletely
-      return
-    end
-     
-    begin
-      while true do
-	# Invariant: At this point, @rest always starts with an
-	# unseen <s> tag.
-	
-	# First, we continue reading until we find the closing </s>
-	# No exception should occur in this loop if we're parsing
-	# a valid XML document.
-	while @rest !~ /^(.*<\/s>)(.*)/m do
-	  @rest = @rest << @file.readline()
-	end
-
-	element = $1
-	@rest = $2
-
-	yield(element) # change HERE: element not parsed!
-
-	# Read on up to the next <s>
-	while @rest !~ /(.*)(<s\s.*)/m do
-	  @rest = @rest << @file.readline()
-	end
-
-	@rest = $2
-      end
-    rescue EOFError
-      @tail = @rest
-      @readCompletely = true
-    end
-  end
-
-  # KE 5.11.03: get_rest: read all of the file not processed up to this point
-  # and return it as a string
-  def get_rest()
-    begin
-      while true do
-	@rest = @rest << @file.readline()
-      end
-    rescue EOFError
-      @readCompletely = true
-    end
-    return @rest
-  end
-end

data/lib/common/RegXML.rb

@@ -1,269 +0,0 @@
-# RegXML
-#
-# Katrin Erk June 2005
-
-# SalsaTigerRegXML: take control of the data structure, no underlying xml
-# representation anymore, re-generation of xml on demand
-
-class RegXML
-
-  def initialize(string, # string representing a single XML element
-                 i_am_text = false) # boolean: xml element (false) or text (true)
-
-    unless string.class == String
-      raise "First argument to RegXML.new must be string. I got #{string.class.to_s}"
-    end
-    if i_am_text
-      @s = string
-      @i_am_text = true
-    else
-      @s = string.gsub(/\n/,  " ").freeze
-      @i_am_text = false
-
-      element_test()
-      dyck_test()
-    end
-  end
-
-  def to_s()
-    return xml_readable(@s)
-  end
-  
-  def text?
-    return @i_am_text
-  end
-
-  def name()
-    if @i_am_text
-      # text
-      return nil
-
-    else
-      # xml element
-      if @s =~ /^\s*<\s*([\w-]+)[\s\/>]/
-        return $1
-      else
-        raise "Cannot parse:\n#{xml_readable(@s)}"
-      end
-    end
-  end
-
-  def attributes()
-    if @i_am_text
-      # text
-      return {}
-
-    else
-      #  xml element
-
-      # remove <element_name  from the beginning of @s,
-      # place the rest up to the first > into elt_contents:
-      # this is a string of the form
-      # - either (name=value)*
-      # - or     (name=value)*/
-      unless @s =~ /^\s*<\s*#{name()}(.*)$/
-        raise "Cannot parse:\n #{xml_readable(@s)}"
-      end
-      
-      retv = Hash.new
-      elt_contents = $1
-      
-      # repeat until only > or /> is left
-      while elt_contents !~ /^\s*\/?>/
-        
-        # shave off the next name=value pair
-        # put the rest into elt_contents
-        # make sure that if the value is quoted with ',
-        # we accept " inside the value, and vice versa.
-        unless elt_contents =~ /^\s*([\w-]+)=(['"])(.*?)\2(.*)$/
-          raise "Cannot parse:\n #{xml_readable(elt_contents)}"
-        end
-        retv[$1] = $3
-        elt_contents = $4
-      end
-
-      return retv
-    end
-  end
-
-  def children_and_text()
-    if @i_am_text
-      return []
-
-    else
-      if unary_element()
-        # <bla/>, no children
-        return []
-      end
-      
-      # @s has the form <bla...>  ... </bla>.
-      # remove <bla ...>  from the beginning of @s,
-      # place the rest up to </bla> into children_s:
-      
-      mainname = name()
-      unless @s =~ /^\s*<\s*#{mainname}(\s+[\w-]+=(["']).*?\2)*\s*>(.*?)<\/\s*#{mainname}\s*>\s*$/
-        raise "Cannot parse:\n #{xml_readable(@s)}"
-      end
-      
-      retv = Array.new
-      children_s = $3
-
-      # repeat until only whitespace is left
-      while children_s !~ /^\s*$/
-        
-      # shave off the next bit of text
-        # put the rest into children_s
-        unless children_s =~ /^\s*(.*?)(<.*$|$)/
-          $stderr.puts "Whole was:\n #{xml_readable(@s)}"
-          $stderr.puts 
-          raise "Cannot parse:\n #{xml_readable(children_s)}"
-        end
-        unless $1.strip.empty?
-          children_s = $2
-          retv << RegXML.new($1, true)
-        end
-        
-        # anything left after we've parsed text?
-        if children_s =~ /^s*$/
-          break
-        end
-
-        # shave off the next child
-        # and put the rest into children_s
-
-        # determine the next child's name, and the string index at which
-        # the element start tag ends with either / or >
-        unless children_s =~ /^\s*(<\s*([\w-]+)(\s+[\w-]+=(["']).*?\4)*\s*)/
-          $stderr.puts "Whole was:\n #{xml_readable(@s)}"
-          $stderr.puts 
-          raise "Cannot parse:\n #{xml_readable(children_s)}"
-        end
-        childname = $2
-        child = $1
-        endofelt_ix = $&.length()
-
-        
-        # and remove it
-        case children_s[endofelt_ix..-1]
-        when /^\/>(.*)$/
-          # next child is a unary element
-          children_s = $1
-          retv << RegXML.new(child + "/>")
-
-        when /^(>.*?<\s*\/\s*#{childname}\s*>)(.*)$/
-          children_s = $2
-          retv << RegXML.new(child + $1)
-
-        else
-          $stderr.puts "Whole was:\n #{xml_readable(@s)}"
-          $stderr.puts 
-          raise "Cannot parse:\n#{xml_readable(children_s)}"
-        end
-      end
-
-      return retv
-    end
-  end
-
-  def RegXML.test()
-    bla = RegXML.new("  <bla blupp='a\"b' 
-lalala=\"c\">
-  <lalala> </lalala>
-  texttext
-  <lala blupp='b'/>
-  nochtext
-  <la> <l/> </la>  
-</ bla >
-")
-    puts "name " + bla.name()
-    puts
-    puts bla.to_s()
-    puts
-    bla.attributes.each { |attr, val|
-      puts "attr " + attr + "=" + val
-    }
-    puts
-    bla.children_and_text.each { |child_obj|
-      if child_obj.text?
-        puts "da text " + child_obj.to_s
-      else
-        puts "da child " + child_obj.to_s
-      end
-    }
-    puts
-
-    puts "NEU"
-    bla = RegXML.new("  < bla blupp='a\"'/> ")
-    puts "name " + bla.name()
-    puts
-    puts bla.to_s()
-    puts
-    bla.attributes.each { |attr, val|
-      puts "attr " + attr + "=" + val
-    }
-    puts
-    bla.children_and_text.each { |child_obj|
-      if child_obj.text?
-        puts "da text " + child_obj.to_s
-      else
-        puts "da child " + child_obj.to_s
-      end
-    }
-    puts
-
-  end
-
-  ##############
-  protected
-
-  def unary_element()
-    # <bla/>
-    if @s =~ /^\s*<.*\/>\s*$/
-      return true
-    else
-      return false
-    end
-  end
-
-  def element_test()
-    # make sure we have a single XML element, either <bla/> or
-    # <bla>...</bla>
-    
-    if unary_element()
-      # <bla/>
-    elsif @s =~ /^\s*<\s*([\w-]+)\W.*?<\/\s*\1\s*>\s*$/
-      # <bla  > ... </bla>
-    else
-      raise "Cannot parse:\n #{xml_readable(@s)}"
-    end
-  end
-
-  def dyck_test()
-    # every prefix of @s must have at least as many < as >
-    opening = 0
-    closing = 0
-    @s.scan(/[<>]/) { |bracket|
-      case bracket
-      when "<"
-        opening += 1
-      when ">"
-        closing += 1
-        if closing > opening
-          raise "More closing than opening brackets in prefix of:\n #{xml_readable(@s)}"
-        end
-      end
-    }
-
-    # and in total, @s must have equally many < and >
-    unless @s.count("<") == @s.count(">")
-      raise "Inequal number of brackets in:\n #{xml_readable(@s)}"
-    end
-  end
-
-  def xml_readable(string)
-    return string.gsub(/>/, ">\n")
-  end
-end
-
-# RegXML.test()
-