shalmaneser 0.0.1.alpha

data/lib/common/TntInterface.rb

@@ -0,0 +1,44 @@
+require "tempfile"
+require "common/AbstractSynInterface"
+
+################################################
+# Interface class
+class TntInterface < SynInterfaceTab
+  TntInterface.announce_me()
+
+  def TntInterface.system()
+    return "tnt"
+  end
+
+  def TntInterface.service()
+    return "pos_tagger"
+  end
+
+  def process_file(infilename,   # string: name of input file
+		   outfilename)  # string: name of output file
+
+    tempfile = Tempfile.new("Tnt")
+    TntInterface.fntab_words_to_file(infilename, tempfile)
+    tempfile.close
+
+    # 1. use grep to remove commentaries from file      
+    # 2. use sed to extract tags tag list:
+    #    - match one or more non-spaces
+    #    - match one or more spaces
+    #    - match one or more non-spaces and write to outfilename 
+    
+    # This assumes that the experiment file entry for pos_tagger_path
+    # has the form 
+    # pos_tagger_path = <program_name> <model>
+
+    Kernel.system(@program_path + " " + tempfile.path +
+		  ' | grep -v -E "^%%" |  sed -e\'s/^[^ ]\{1,\}[[:space:]]\{1,\}\([^ ]\{1,\}\)/\1/\' > '+outfilename)
+
+    tempfile.close(true) # delete tempfile
+    unless `cat #{infilename} | wc -l`.strip ==
+                                     `cat #{outfilename} | wc -l`.strip
+      raise "Error: tagged file has different line number from corpus file!"
+    end   
+  end
+end
+

data/lib/common/Tree.rb

@@ -0,0 +1,61 @@
+require 'common/Graph'
+
+class TreeNode < GraphNode
+
+  def initialize(id)
+    super(id)
+  end
+
+  # redo the ancestor-related methods,
+  # since here we only have one parent per node
+  def parent()
+    retv = parents()
+    if retv.nil?
+      return nil
+    else
+      return retv.first
+    end
+  end
+
+  def parent_label()
+    retv = parent_labels()
+    if retv.nil?
+      return nil
+    else
+      return retv.first
+    end
+  end
+ 
+
+  def parent_with_edgelabel()
+    retv = parents_with_edgelabel()
+
+    if retv.nil?
+      return nil
+    else
+      return retv.first
+    end
+  end
+
+
+  def add_parent(parent, edgelabel, varhash={})
+    set_parent(parent, edgelabel, varhash)
+  end
+
+  def set_parent(parent, edgelabel, varhash={})
+    # remove old parent
+    each_parent_with_edgelabel { |label, parent|
+      remove_parent(parent, label, varhash)
+    }
+
+    # set new parent
+    @parents << [edgelabel, parent]
+
+    # and vice versa: add self as child to parent
+    unless varhash["pointer_insteadof_edge"]
+      unless parent.children_with_edgelabel().include? [edgelabel, self]
+        parent.add_child(self, edgelabel)
+      end
+    end
+  end
+end

data/lib/common/TreetaggerInterface.rb

@@ -0,0 +1,303 @@
+# sp 30 11 06
+# extended by TreeTaggerPOSInterface
+
+require "tempfile"
+
+require "common/AbstractSynInterface"
+
+###########
+# KE dec 7, 06
+# common mixin for both Treetagger modules, doing the actual processing
+module TreetaggerModule
+  ###
+  # Treetagger does both lemmatization and POS-tagging.
+  # However, the way the SynInterface system is set up in Shalmaneser,
+  # each SynInterface can offer only _one_ service.
+  # This means that we cannot do a SynInterface that writes
+  # both a POS file and a lemma file.
+  # Instead, both will include this module, which does the
+  # actual TreeTagger call and then stores the result in a file
+  # of its own, similar to the 'outfilename' given to TreetaggerInterface.process_file
+  # but with a separate extension.
+  # really_process_file checks for existence of this file because,
+  # if the TreeTagger lemmatization and POS-tagging classes are called separately,
+  # one of them will go first, and the 2nd one will not need to do the 
+  # TreeTagger call anymore
+  #
+  # really_process_file returns a filename, the name of the file containing
+  # the TreeTagger output with both POS tags and lemma information
+  #
+  # WARNING: this method assumes that outfilename contains a suffix
+  # that can be replaced by .TreeTagger
+  def really_process_file(infilename, # string: name of input file
+                          outfilename,# string: name of file that the caller is to produce
+                          make_new_outfile_anyway = false) # Boolean: run TreeTagger in any case?
+
+    # fabricate the filename in which the 
+    # actual TreeTagger output will be placed:
+    # <directory> + <outfilename minus last suffix> + ".TreeTagger"
+    current_suffix = outfilename[outfilename.rindex(".")..-1]
+    my_outfilename = File.dirname(outfilename) + "/" + 
+      File.basename(outfilename, current_suffix) + 
+      ".TreeTagger"
+
+    ##
+    # does it exist? then just return it
+    if not(make_new_outfile_anyway) and File.exists?(my_outfilename)
+      return my_outfilename
+    end
+
+    ##
+    # else construct it, then return it
+    tempfile = Tempfile.new("Treetagger")
+    TreetaggerInterface.fntab_words_to_file(infilename, tempfile, "<EOS>", "iso")
+    tempfile.close
+
+    # call TreeTagger
+    Kernel.system(@program_path+" "+tempfile.path + 
+                  " > " + my_outfilename)
+    tempfile.close(true) # delete first tempfile
+    
+    # external problem: sometimes, the treetagger keeps the last <EOS> for itself, 
+    # resulting on a .tagged file missing the last (blank) line
+    
+    original_length = `cat #{infilename} | wc -l`.strip.to_i
+    puts infilename
+    lemmatised_length = `cat #{my_outfilename} | wc -l`.strip.to_i
+
+#    `cp #{tempfile2.path()} /tmp/lout`
+
+    case original_length - lemmatised_length
+    when 0
+      # everything ok, don't do anything
+    when 1
+      # add one more newline to the .tagged file
+      `echo "" >> #{my_outfilename}`
+    else
+      # this is "real" error
+      STDERR.puts "Original length: #{original_length}\tLemmatised length: #{lemmatised_length}"
+      STDERR.puts "Error: lemmatiser/tagger output for for #{File.basename(infilename)}"
+      $stderr.puts "has different line number from corpus file!"
+      raise
+    end
+    
+
+    return my_outfilename
+  end
+end
+
+#######################################
+class TreetaggerInterface < SynInterfaceTab
+  TreetaggerInterface.announce_me()
+  
+  include TreetaggerModule
+
+  ###
+  def TreetaggerInterface.system()
+    return "treetagger"
+  end
+
+  ###
+  def TreetaggerInterface.service()
+    return "lemmatizer"
+  end
+
+  ###
+  # convert TreeTagger's penn tagset into Collins' penn tagset *argh*
+  
+  def convert_to_berkeley(line)
+      line.chomp!
+      return line.gsub(/\(/,"-LRB-").gsub(/\)/,"-RRB-").gsub(/''/,"\"").gsub(/\`\`/,"\"")
+  end
+  
+
+  ###
+  def process_file(infilename,  # string: name of input file
+                   outfilename) # string: name of output file
+
+    # KE change here
+    ttfilename = really_process_file(infilename, outfilename)
+
+    # write all output to tempfile2 first, then 
+    # change ISO to UTF-8 into outputfile
+    tempfile2 = Tempfile.new("treetagger")
+    tempfile2.close()
+    
+    # 2. use cut to get the actual lemmtisation
+    
+    Kernel.system("cat " + ttfilename + 
+		  ' | sed -e\'s/<EOS>//\' | cut -f3 > '+tempfile2.path()) 
+    
+    # transform ISO-8859-1 back to UTF-8, 
+    # write to 'outfilename'    
+    begin
+      outfile = File.new(outfilename, "w")
+    rescue
+      raise "Could not write to #{outfilename}"
+    end
+    tempfile2.open
+    # AB: Internally all the flow is an utf-8 encoded stream.
+    # TreeTagger consumes one byte encodings (but we should provide a
+    # utf-8 model for German). So we convert utf-8 to latin1, then
+    # process the text and convert it back to utf-8.
+    #
+    while line = tempfile2.gets
+	#outfile.puts UtfIso.from_iso_8859_1(line)
+      utf8line = UtfIso.from_iso_8859_1(line)
+      outfile.puts convert_to_berkeley(utf8line)
+    end
+    
+    # remove second tempfile, finalize output file
+    tempfile2.close(true)
+    outfile.close()
+
+  end
+end
+
+
+# sp 30 11 06
+#
+# using TreeTagger for POS tagging of English text
+#
+# copy-and-paste from lemmatisation
+#
+# differences: 
+# 1. use field 2 and not 3 from the output
+# 2. convert tags from what Treetagger thinks is the Penn Tagset to what TnT and Collins think is the Penn Tagset
+# 
+# KE 7 12 06
+# change interface such that TreeTagger is called only once
+# and both POS tags and lemma are read from the same files,
+# rather than calling the tagger twice
+class TreetaggerPOSInterface < SynInterfaceTab
+  TreetaggerPOSInterface.announce_me()
+  include TreetaggerModule
+  
+  ###
+  def TreetaggerPOSInterface.system()
+    return "treetagger"
+  end
+
+  ###
+  def TreetaggerPOSInterface.service()
+    return "pos_tagger"
+  end
+
+  ###
+  # convert TreeTagger's penn tagset into Collins' penn tagset *argh*
+
+  def convert_to_collins(line)
+    line.chomp!
+    return line.gsub(/^PP/,"PRP").gsub(/^NP/,"NNP").gsub(/^VV/,"VB").gsub(/^VH/,"VB").gsub(/^SENT/,".")
+  end
+
+  ###
+  def process_file(infilename,  # string: name of input file
+                   outfilename) # string: name of output file
+
+    # KE change here
+    tt_filename = really_process_file(infilename, outfilename, true)
+
+    # write all output to tempfile2 first, then 
+    # change ISO to UTF-8 into outputfile
+    tempfile2 = Tempfile.new("treetagger")
+    tempfile2.close()
+    
+    # 2. use cut to get the actual lemmtisation
+
+    Kernel.system("cat " + tt_filename +
+		  ' | sed -e\'s/<EOS>//\' | cut -f2 > '+tempfile2.path()) 
+    
+    # transform ISO-8859-1 back to UTF-8, 
+    # write to 'outfilename'    
+    begin
+      outfile = File.new(outfilename, "w")
+    rescue
+      raise "Could not write to #{outfilename}"
+    end
+    tempfile2.open()
+    while (line = tempfile2.gets())
+      outfile.puts UtfIso.from_iso_8859_1(convert_to_collins(line))
+    end
+    
+    # remove second tempfile, finalize output file
+    tempfile2.close(true)
+    outfile.close()
+  end
+end
+
+###############
+# an interpreter that only has Treetagger, no parser
+class TreetaggerInterpreter < SynInterpreter
+  TreetaggerInterpreter.announce_me()
+
+  ###
+  # names of the systems interpreted by this class:
+  # returns a hash service(string) -> system name (string),
+  # e.g.
+  # { "parser" => "collins", "lemmatizer" => "treetagger" }
+  def TreetaggerInterpreter.systems()
+    return {
+      "pos_tagger" => "treetagger",
+    }
+  end
+
+  ###
+  # names of additional systems that may be interpreted by this class
+  # returns a hash service(string) -> system name(string)
+  # same as names()
+  def TreetaggerInterpreter.optional_systems()
+    return {
+      "lemmatizer" => "treetagger"
+    }
+  end
+
+  ###
+  # generalize over POS tags.
+  #
+  # returns one of:
+  #
+  # adj:  adjective (phrase)
+  # adv:  adverb (phrase)
+  # card: numbers, quantity phrases
+  # con:  conjunction
+  # det:  determiner, including possessive/demonstrative pronouns etc.
+  # for:  foreign material
+  # noun: noun (phrase), including personal pronouns, proper names, expletives
+  # part: particles, truncated words (German compound parts)
+  # prep: preposition (phrase)
+  # pun:  punctuation, brackets, etc.
+  # sent: sentence
+  # top:  top node of a sentence
+  # verb: verb (phrase)
+  # nil:  something went wrong
+  #
+  # returns: string, or nil
+  def TreetaggerInterpreter.category(node) # SynNode
+    pt = TreetaggerInterpreter.pt(node)
+    if pt.nil?
+      # phrase type could not be determined
+      return nil
+    end
+
+    pt.to_s.strip() =~ /^([^-]*)/  
+    case $1
+    when  /^JJ/ ,/(WH)?ADJP/, /^PDT/ then  return "adj"
+    when /^RB/, /(WH)?ADVP/, /^UH/ then return "adv"
+    when /^CD/, /^QP/ then  return "card"
+    when /^CC/, /^WRB/, /^CONJP/ then return "con"
+    when /^DT/, /^POS/ then  return "det"
+    when /^FW/, /^SYM/ then  return "for"
+    when /^N/, "WHAD", "WDT", /^PRP/ , /^WHNP/, /^EX/, /^WP/  then return "noun"
+    when  /^IN/ , /^TO/, /(WH)?PP/, "RP", /^PR(T|N)/ then return "prep"
+    when /^PUNC/, /LRB/, /RRB/, /[,'".:;!?\(\)]/ then  return "pun"
+    when /^S(s|bar|BAR|G|Q|BARQ|INV)?$/, /^UCP/, /^FRAG/, /^X/, /^INTJ/ then return "sent"
+    when /^TOP/ then  return "top"
+    when /^TRACE/ then  return "trace"
+    when /^V/ , /^MD/ then return "verb"
+    else
+#      $stderr.puts "WARNING: Unknown category/POS "+c.to_s + " (English data)"
+      return nil
+    end
+  end
+end

data/lib/common/headz.rb

@@ -0,0 +1,338 @@
+# name: Module Headz
+# auth: albu@coli.uni-sb.de
+# 
+# modified KE Sept 04:
+# changed from old Sentence pkg to new SalsaTigerSentence pkg
+#
+# modified KE April 05:
+# suppress the flood of warnings
+#
+# modified SP June 05: added some more cases; change to SalsTigerRegXML
+# 
+# 
+# INIT: REXML TIGER sentence, 
+# FUNC: syn_nodes(term/non_term) -> heads   
+#  
+# 
+# usage:
+# 
+# h = Headz.new() 
+#
+# hash = h.get_sem_head(node) # node is a SalsaTigerXmlNode obj
+# 
+# head = hash["head"]
+# prep = hash["prep"]
+# 
+# if h.complex(head)
+#   print "preposition of conjunction involved"
+# end
+
+require "common/SalsaTigerRegXML"
+
+class Headz 
+
+  def initialize()
+    @Helpers = HeadzHelpers.new()
+    @Verbose = false #KE 13.4.05: please not that many messages!
+  end
+  
+  # head of one node
+  def get_sem_head(node)
+    gsh(node)
+  end
+  
+  # all headz of top-nodes covering fe
+  def get_fe_heads(fe)
+    if (const = fe.children())
+      const.map { |node|
+	get_sem_head(node)
+      }
+    else
+      $stderr.puts "Headz.get_sem_head: no children for FE #{fe}"
+      []
+    end
+  end
+  
+  def gsh (node)
+    if !node then 
+      if @Verbose then $stderr.puts "Headz.gsh: no input node" end
+      return {}
+
+    elsif node.is_terminal? then return Hash['head'=>node]
+
+    else
+      case node.category
+      when 'AP'
+	return gsh(@Helpers.get_dtr(node,'HD'))
+
+      when 'AVP'
+	return gsh(@Helpers.get_dtr(node,'HD'))
+      when 'CAP', 'CAVP', 'CNP', 'CPP', 'CS', 'CVP'
+	conjs = @Helpers.get_conjuncts(node)
+	head = gsh(conjs.shift)
+        if head
+          head.update(Hash["conj"=>gsh_conjs(conjs)])
+        end
+        return head
+
+      when 'NM'
+	return gsh(@Helpers.get_rightmost_dtr(node,'NMC'))
+      when 'NP'
+        nk = @Helpers.get_rightmost_dtr(node,'NK')
+        if nk
+          return gsh(nk)
+        else
+          return gsh(@Helpers.get_rightmost_dtr(node, "NN"))
+        end
+
+      when 'PN'
+	pncs = @Helpers.get_dtrs(node,'PNC')
+	head = gsh(pncs.last)
+        if head
+          head.update(Hash["pncs"=>pncs])
+        end
+        return head
+
+      when 'PP'
+	return pp(node)
+
+      when 'S'
+	return s(node)
+      when 'VROOT'
+        dtrs = @Helpers.get_dtrs(node,'--')
+
+        # discourse level node with sentence nodes below?
+        # or conjunction with sentence nodes below?
+        discourselevel_dtr = dtrs.detect { |n| n.category == "DL"}
+        co_dtr = dtrs.detect { |n| n.category == "CO" }
+        if discourselevel_dtr
+          dtrs = discourselevel_dtr.children()
+        elsif co_dtr
+          dtrs = co_dtr.children()
+        end
+
+
+        # take first sentence node
+        sent_dtr = dtrs.detect {|n| n.category =~ /^C?S/}
+        if sent_dtr
+          return gsh(sent_dtr)
+        else          
+#          $stderr.puts "headz Warning: no sentence found below VROOT! Node #{node.id()}"
+          return nil
+        end
+
+      when 'VP'
+	return vp(node)
+
+      when 'MTA'
+        return gsh(@Helpers.get_rightmost_dtr(node,'ADC'))
+
+      when 'VZ'
+	return gsh(@Helpers.get_dtr(node,'HD'))
+      else
+	if @Verbose 
+	  $stderr.puts " Headz.gsh: no rule for #{node.category}" 
+	end
+	{}
+      end
+    end
+  end
+  
+  # flatten the processed conjs to a list of (head) Hashes 
+  # containing no conj features themselves 
+  def gsh_conjs(conjs)
+    flat = Array.new
+    
+    conjs.each {|conj|
+      current = gsh(conj)
+      @Helpers.descend(current,flat)
+    }
+    
+    flat
+  end
+ 
+  #####################################3
+  def pp(node)
+
+    prep = node.terminals_sorted().detect { |n| 
+      (pt = n.part_of_speech()) and 
+        (pt =~ /^APPR/ or 
+           pt =~ /^PWAV/ or
+           pt =~ /^C?PP/
+         )
+    }
+
+    if (lastnk = @Helpers.get_rightmost_dtr(node,'NK'))
+      head = gsh(lastnk)
+      if head and prep
+        head.update(Hash['prep'=>prep])      
+      end
+
+    elsif (re = @Helpers.get_dtr(node,'RE'))
+      head = gsh(re)
+      if head and prep
+        head.update(Hash['prep'=>prep])      
+      end
+    else
+      if @Verbose then $stderr.puts " pp: no rule for #{node}" end
+    end
+
+    head
+  end
+  
+  ################
+  def s(node)
+    head = @Helpers.get_dtr(node,'HD')
+    if !head
+#      $stderr.puts " s: no head for #{node}"
+      return Hash[]
+    end
+    
+    if head.outdeg() == 0
+      return gsh(head)
+    end
+
+    oc = @Helpers.get_dtr(node,'OC')
+    case head.category
+    when 'VVFIN' 
+      if svp = @Helpers.get_dtr(node,'SVP') then 
+        h = gsh(head)
+        if h
+          return h.update(Hash['svp'=>gsh(svp), 'oc'=>gsh(oc)]) 
+        else
+          return h
+        end
+      else 
+        return gsh(head)
+      end
+
+    when 'VAFIN'
+      if oc && headd = @Helpers.get_dtr(oc,'HD')
+        h = gsh(headd)
+        if h
+          return h.update(Hash['oc'=>gsh(oc)])
+        else
+          return h
+        end
+	
+      elsif pd = @Helpers.get_dtr(node,'PD') && head = @Helpers.get_dtr(pd,'HD')
+        return gsh(head)
+      
+      else 
+        if @Verbose then $stderr.puts " s: no rule for #{node}" end
+      end
+    else
+      if @Verbose then $stderr.puts " s: no rule for #{node}" end
+    end  
+  end
+  
+  ################
+  def vp(node)
+    head = gsh(@Helpers.get_dtr(node,'HD'))
+    tmp = @Verbose
+    @Verbose = false
+    newHash = Hash.new
+    ["da","oa"].each { |type| 
+      if (dtr = @Helpers.get_dtr(node,type.upcase))
+	newHash[type] = gsh(dtr)
+      end
+    }
+    @Verbose = tmp
+    if head 
+      return head.update(newHash) 
+    else 
+      return newHash 
+    end
+  end
+  
+  ################
+  # Access
+  def head(h)
+    return h['head']
+  end
+
+  def complex(h)
+    prep(h) or conj(h)
+  end
+
+  def prep(h)
+    return h['prep']
+  end
+
+  def conj(h)
+    return h['conj']
+  end
+
+
+  
+end # Class Headz
+
+
+class HeadzHelpers
+  @Verbose = true
+  
+  # Conjunction
+  
+  def get_conjuncts(node)
+    conjuncts = get_dtrs(node,'CJ')
+  end
+
+  # flatten
+  def descend(current,flat)
+    if current.nil?
+      return flat
+    end
+
+    if current.has_key?("conj") then
+      tmp = current.delete("conj")
+      flat.push current
+      tmp.each {|item|     
+	descend(item,flat)}
+    else 
+      flat.push current 
+    end
+  end
+  
+  # Zugriff
+  
+  def get_dtr(node,label)
+    if (dtrs = node.children_by_edgelabels([label]))
+      dtrs.first
+    else
+      if @Verbose then $stderr.puts " SelectHeadDtr: no #{label} dtr for #{node}" end
+      nil
+    end
+  end
+  
+  def get_dtrs(node,label)
+    if ! dtrs = node.children_by_edgelabels([label])
+      if @Verbose then $stderr.puts " SelectHeadDtr: no #{label} dtr for #{node}" end
+    else
+      dtrs
+    end
+  end
+  
+  def get_rightmost_dtr(node,label)
+    children = node.children_by_edgelabels([label])
+    if re = children.last then re
+    else 
+      if @Verbose then $stderr.puts " SelectHeadDtr: no #{label} dtrs for #{node}" end
+      nil 
+    end
+  end
+
+#   def l2h(list)
+#     h = Hash.new
+#     while (list.length > 1) do
+#       h[list.shift] = list.shift
+#     end
+#     if list.length == 1 then 
+#       $stderr.puts "l2h: odd number of elems: " + list.join(" / ")
+#     end
+#     h
+#   end
+  
+end # Class HeadzHelpers
+
+
+