frprep 0.0.1.prealpha

data/lib/frprep/TntInterface.rb

@@ -0,0 +1,44 @@
+require "tempfile"
+require "frprep/AbstractSynInterface"
+
+################################################
+# Interface class
+class TntInterface < SynInterfaceTab
+  TntInterface.announce_me()
+
+  def TntInterface.system()
+    return "tnt"
+  end
+
+  def TntInterface.service()
+    return "pos_tagger"
+  end
+
+  def process_file(infilename,   # string: name of input file
+		   outfilename)  # string: name of output file
+
+    tempfile = Tempfile.new("Tnt")
+    TntInterface.fntab_words_to_file(infilename, tempfile)
+    tempfile.close
+
+    # 1. use grep to remove commentaries from file      
+    # 2. use sed to extract tags tag list:
+    #    - match one or more non-spaces
+    #    - match one or more spaces
+    #    - match one or more non-spaces and write to outfilename 
+    
+    # This assumes that the experiment file entry for pos_tagger_path
+    # has the form 
+    # pos_tagger_path = <program_name> <model>
+
+    Kernel.system(@program_path + " " + tempfile.path +
+		  ' | grep -v -E "^%%" |  sed -e\'s/^[^ ]\{1,\}[[:space:]]\{1,\}\([^ ]\{1,\}\)/\1/\' > '+outfilename)
+
+    tempfile.close(true) # delete tempfile
+    unless `cat #{infilename} | wc -l`.strip ==
+                                     `cat #{outfilename} | wc -l`.strip
+      raise "Error: tagged file has different line number from corpus file!"
+    end   
+  end
+end
+

data/lib/frprep/Tree.rb

@@ -0,0 +1,61 @@
+require 'frprep/Graph'
+
+class TreeNode < GraphNode
+
+  def initialize(id)
+    super(id)
+  end
+
+  # redo the ancestor-related methods,
+  # since here we only have one parent per node
+  def parent()
+    retv = parents()
+    if retv.nil?
+      return nil
+    else
+      return retv.first
+    end
+  end
+
+  def parent_label()
+    retv = parent_labels()
+    if retv.nil?
+      return nil
+    else
+      return retv.first
+    end
+  end
+ 
+
+  def parent_with_edgelabel()
+    retv = parents_with_edgelabel()
+
+    if retv.nil?
+      return nil
+    else
+      return retv.first
+    end
+  end
+
+
+  def add_parent(parent, edgelabel, varhash={})
+    set_parent(parent, edgelabel, varhash)
+  end
+
+  def set_parent(parent, edgelabel, varhash={})
+    # remove old parent
+    each_parent_with_edgelabel { |label, parent|
+      remove_parent(parent, label, varhash)
+    }
+
+    # set new parent
+    @parents << [edgelabel, parent]
+
+    # and vice versa: add self as child to parent
+    unless varhash["pointer_insteadof_edge"]
+      unless parent.children_with_edgelabel().include? [edgelabel, self]
+        parent.add_child(self, edgelabel)
+      end
+    end
+  end
+end

data/lib/frprep/TreetaggerInterface.rb

@@ -0,0 +1,303 @@
+# sp 30 11 06
+# extended by TreeTaggerPOSInterface
+
+require "tempfile"
+
+require "frprep/AbstractSynInterface"
+
+###########
+# KE dec 7, 06
+# common mixin for both Treetagger modules, doing the actual processing
+module TreetaggerModule
+  ###
+  # Treetagger does both lemmatization and POS-tagging.
+  # However, the way the SynInterface system is set up in Shalmaneser,
+  # each SynInterface can offer only _one_ service.
+  # This means that we cannot do a SynInterface that writes
+  # both a POS file and a lemma file.
+  # Instead, both will include this module, which does the
+  # actual TreeTagger call and then stores the result in a file
+  # of its own, similar to the 'outfilename' given to TreetaggerInterface.process_file
+  # but with a separate extension.
+  # really_process_file checks for existence of this file because,
+  # if the TreeTagger lemmatization and POS-tagging classes are called separately,
+  # one of them will go first, and the 2nd one will not need to do the 
+  # TreeTagger call anymore
+  #
+  # really_process_file returns a filename, the name of the file containing
+  # the TreeTagger output with both POS tags and lemma information
+  #
+  # WARNING: this method assumes that outfilename contains a suffix
+  # that can be replaced by .TreeTagger
+  def really_process_file(infilename, # string: name of input file
+                          outfilename,# string: name of file that the caller is to produce
+                          make_new_outfile_anyway = false) # Boolean: run TreeTagger in any case?
+
+    # fabricate the filename in which the 
+    # actual TreeTagger output will be placed:
+    # <directory> + <outfilename minus last suffix> + ".TreeTagger"
+    current_suffix = outfilename[outfilename.rindex(".")..-1]
+    my_outfilename = File.dirname(outfilename) + "/" + 
+      File.basename(outfilename, current_suffix) + 
+      ".TreeTagger"
+
+    ##
+    # does it exist? then just return it
+    if not(make_new_outfile_anyway) and File.exists?(my_outfilename)
+      return my_outfilename
+    end
+
+    ##
+    # else construct it, then return it
+    tempfile = Tempfile.new("Treetagger")
+    TreetaggerInterface.fntab_words_to_file(infilename, tempfile, "<EOS>", "iso")
+    tempfile.close
+
+    # call TreeTagger
+    Kernel.system(@program_path+" "+tempfile.path + 
+                  " > " + my_outfilename)
+    tempfile.close(true) # delete first tempfile
+    
+    # external problem: sometimes, the treetagger keeps the last <EOS> for itself, 
+    # resulting on a .tagged file missing the last (blank) line
+    
+    original_length = `cat #{infilename} | wc -l`.strip.to_i
+    puts infilename
+    lemmatised_length = `cat #{my_outfilename} | wc -l`.strip.to_i
+
+#    `cp #{tempfile2.path()} /tmp/lout`
+
+    case original_length - lemmatised_length
+    when 0
+      # everything ok, don't do anything
+    when 1
+      # add one more newline to the .tagged file
+      `echo "" >> #{my_outfilename}`
+    else
+      # this is "real" error
+      STDERR.puts "Original length: #{original_length}\tLemmatised length: #{lemmatised_length}"
+      STDERR.puts "Error: lemmatiser/tagger output for for #{File.basename(infilename)}"
+      $stderr.puts "has different line number from corpus file!"
+      raise
+    end
+    
+
+    return my_outfilename
+  end
+end
+
+#######################################
+class TreetaggerInterface < SynInterfaceTab
+  TreetaggerInterface.announce_me()
+  
+  include TreetaggerModule
+
+  ###
+  def TreetaggerInterface.system()
+    return "treetagger"
+  end
+
+  ###
+  def TreetaggerInterface.service()
+    return "lemmatizer"
+  end
+
+  ###
+  # convert TreeTagger's penn tagset into Collins' penn tagset *argh*
+  
+  def convert_to_berkeley(line)
+      line.chomp!
+      return line.gsub(/\(/,"-LRB-").gsub(/\)/,"-RRB-").gsub(/''/,"\"").gsub(/\`\`/,"\"")
+  end
+  
+
+  ###
+  def process_file(infilename,  # string: name of input file
+                   outfilename) # string: name of output file
+
+    # KE change here
+    ttfilename = really_process_file(infilename, outfilename)
+
+    # write all output to tempfile2 first, then 
+    # change ISO to UTF-8 into outputfile
+    tempfile2 = Tempfile.new("treetagger")
+    tempfile2.close()
+    
+    # 2. use cut to get the actual lemmtisation
+    
+    Kernel.system("cat " + ttfilename + 
+		  ' | sed -e\'s/<EOS>//\' | cut -f3 > '+tempfile2.path()) 
+    
+    # transform ISO-8859-1 back to UTF-8, 
+    # write to 'outfilename'    
+    begin
+      outfile = File.new(outfilename, "w")
+    rescue
+      raise "Could not write to #{outfilename}"
+    end
+    tempfile2.open
+    # AB: Internally all the flow is an utf-8 encoded stream.
+    # TreeTagger consumes one byte encodings (but we should provide a
+    # utf-8 model for German). So we convert utf-8 to latin1, then
+    # process the text and convert it back to utf-8.
+    #
+    while line = tempfile2.gets
+	#outfile.puts UtfIso.from_iso_8859_1(line)
+      utf8line = UtfIso.from_iso_8859_1(line)
+      outfile.puts convert_to_berkeley(utf8line)
+    end
+    
+    # remove second tempfile, finalize output file
+    tempfile2.close(true)
+    outfile.close()
+
+  end
+end
+
+
+# sp 30 11 06
+#
+# using TreeTagger for POS tagging of English text
+#
+# copy-and-paste from lemmatisation
+#
+# differences: 
+# 1. use field 2 and not 3 from the output
+# 2. convert tags from what Treetagger thinks is the Penn Tagset to what TnT and Collins think is the Penn Tagset
+# 
+# KE 7 12 06
+# change interface such that TreeTagger is called only once
+# and both POS tags and lemma are read from the same files,
+# rather than calling the tagger twice
+class TreetaggerPOSInterface < SynInterfaceTab
+  TreetaggerPOSInterface.announce_me()
+  include TreetaggerModule
+  
+  ###
+  def TreetaggerPOSInterface.system()
+    return "treetagger"
+  end
+
+  ###
+  def TreetaggerPOSInterface.service()
+    return "pos_tagger"
+  end
+
+  ###
+  # convert TreeTagger's penn tagset into Collins' penn tagset *argh*
+
+  def convert_to_collins(line)
+    line.chomp!
+    return line.gsub(/^PP/,"PRP").gsub(/^NP/,"NNP").gsub(/^VV/,"VB").gsub(/^VH/,"VB").gsub(/^SENT/,".")
+  end
+
+  ###
+  def process_file(infilename,  # string: name of input file
+                   outfilename) # string: name of output file
+
+    # KE change here
+    tt_filename = really_process_file(infilename, outfilename, true)
+
+    # write all output to tempfile2 first, then 
+    # change ISO to UTF-8 into outputfile
+    tempfile2 = Tempfile.new("treetagger")
+    tempfile2.close()
+    
+    # 2. use cut to get the actual lemmtisation
+
+    Kernel.system("cat " + tt_filename +
+		  ' | sed -e\'s/<EOS>//\' | cut -f2 > '+tempfile2.path()) 
+    
+    # transform ISO-8859-1 back to UTF-8, 
+    # write to 'outfilename'    
+    begin
+      outfile = File.new(outfilename, "w")
+    rescue
+      raise "Could not write to #{outfilename}"
+    end
+    tempfile2.open()
+    while (line = tempfile2.gets())
+      outfile.puts UtfIso.from_iso_8859_1(convert_to_collins(line))
+    end
+    
+    # remove second tempfile, finalize output file
+    tempfile2.close(true)
+    outfile.close()
+  end
+end
+
+###############
+# an interpreter that only has Treetagger, no parser
+class TreetaggerInterpreter < SynInterpreter
+  TreetaggerInterpreter.announce_me()
+
+  ###
+  # names of the systems interpreted by this class:
+  # returns a hash service(string) -> system name (string),
+  # e.g.
+  # { "parser" => "collins", "lemmatizer" => "treetagger" }
+  def TreetaggerInterpreter.systems()
+    return {
+      "pos_tagger" => "treetagger",
+    }
+  end
+
+  ###
+  # names of additional systems that may be interpreted by this class
+  # returns a hash service(string) -> system name(string)
+  # same as names()
+  def TreetaggerInterpreter.optional_systems()
+    return {
+      "lemmatizer" => "treetagger"
+    }
+  end
+
+  ###
+  # generalize over POS tags.
+  #
+  # returns one of:
+  #
+  # adj:  adjective (phrase)
+  # adv:  adverb (phrase)
+  # card: numbers, quantity phrases
+  # con:  conjunction
+  # det:  determiner, including possessive/demonstrative pronouns etc.
+  # for:  foreign material
+  # noun: noun (phrase), including personal pronouns, proper names, expletives
+  # part: particles, truncated words (German compound parts)
+  # prep: preposition (phrase)
+  # pun:  punctuation, brackets, etc.
+  # sent: sentence
+  # top:  top node of a sentence
+  # verb: verb (phrase)
+  # nil:  something went wrong
+  #
+  # returns: string, or nil
+  def TreetaggerInterpreter.category(node) # SynNode
+    pt = TreetaggerInterpreter.pt(node)
+    if pt.nil?
+      # phrase type could not be determined
+      return nil
+    end
+
+    pt.to_s.strip() =~ /^([^-]*)/  
+    case $1
+    when  /^JJ/ ,/(WH)?ADJP/, /^PDT/ then  return "adj"
+    when /^RB/, /(WH)?ADVP/, /^UH/ then return "adv"
+    when /^CD/, /^QP/ then  return "card"
+    when /^CC/, /^WRB/, /^CONJP/ then return "con"
+    when /^DT/, /^POS/ then  return "det"
+    when /^FW/, /^SYM/ then  return "for"
+    when /^N/, "WHAD", "WDT", /^PRP/ , /^WHNP/, /^EX/, /^WP/  then return "noun"
+    when  /^IN/ , /^TO/, /(WH)?PP/, "RP", /^PR(T|N)/ then return "prep"
+    when /^PUNC/, /LRB/, /RRB/, /[,'".:;!?\(\)]/ then  return "pun"
+    when /^S(s|bar|BAR|G|Q|BARQ|INV)?$/, /^UCP/, /^FRAG/, /^X/, /^INTJ/ then return "sent"
+    when /^TOP/ then  return "top"
+    when /^TRACE/ then  return "trace"
+    when /^V/ , /^MD/ then return "verb"
+    else
+#      $stderr.puts "WARNING: Unknown category/POS "+c.to_s + " (English data)"
+      return nil
+    end
+  end
+end

data/lib/frprep/do_parses.rb

@@ -0,0 +1,142 @@
+# -*- encoding: utf-8 -*-
+
+# AB, 2010-11-25
+
+##############################
+# class for managing parses:
+#
+# Given either a directory with tab format files or
+# a directory with SalsaTigerXML files (or both) and 
+# a directory for putting parse files:
+# - parse, unless no parsing set in the experiment file
+# - for each parsed file: yield one OneParsedFile object
+require 'frprep/one_parsed_file'
+
+class DoParses
+  def initialize(exp,           # FrPrepConfigData object
+		 file_suffixes, # hash: file type(string) -> suffix(string)
+		 parse_dir,     # string: name of directory to put parses
+		 var_hash = {}) # further directories
+    @exp = exp
+    @file_suffixes = file_suffixes
+    @parse_dir = parse_dir
+    @tab_dir = var_hash["tab_dir"]
+    @stxml_dir = var_hash["stxml_dir"]
+
+    # pre-parsed data available? 
+    @parsed_files = @exp.get("directory_parserout")
+  end
+
+  ###
+  def each_parsed_file()
+    if @exp.get("do_postag") 
+      postag_suffix = @file_suffixes["pos"] 
+    else
+      postag_suffix = nil
+    end
+
+    if @exp.get("do_lemmatize")
+      lemma_suffix = @file_suffixes["lemma"] 
+    else
+      lemma_suffix = nil
+    end
+
+    if @exp.get("do_parse")
+
+      # get parser interface
+      sys_class = SynInterfaces.get_interface("parser", 
+ 					      @exp.get("parser"))
+      unless sys_class
+        raise "Shouldn't be here"
+      end
+      parse_suffix = "." + sys_class.name()
+      sys = sys_class.new(@exp.get("parser_path"),
+ 			  @file_suffixes["tab"],
+ 			  parse_suffix,
+ 			  @file_suffixes["stxml"],
+ 			  "pos_suffix" => postag_suffix,
+ 			  "lemma_suffix" => lemma_suffix,
+ 			  "tab_dir" => @tab_dir)
+
+      if @parsed_files
+        # reuse old parses
+        
+        $stderr.puts "Frprep: using pre-computed parses in " + @parsed_files.to_s()
+        $stderr.puts "Frprep: Postprocessing SalsaTigerXML data"
+        
+        Dir[@parsed_files + "*"].each { |parsefilename|
+          
+          if File.stat(parsefilename).ftype != "file"
+            # something other than a file
+            next
+          end
+          
+          
+          # core filename: remove directory and anything after the last "."
+          filename_core = File.basename(parsefilename, ".*")
+          #print "FN ", filename_core, " PN ", parsefilename, " sys ", sys, "\n"
+          # use iterator to read each parsed file
+          yield OneParsedFile.new(filename_core, parsefilename, sys)
+        }
+
+      else
+        # do new parses
+        $stderr.puts "Frprep: Parsing"
+        
+        # sanity check
+        unless @exp.get("parser_path")
+          raise "Parsing: I need 'parser_path' in the experiment file"
+        end
+        unless @tab_dir
+          raise "Cannot parse without tab files"
+        end
+
+        # parse
+        sys.process_dir(@tab_dir, @parse_dir)
+
+        $stderr.puts "Frprep: Postprocessing SalsaTigerXML data"
+
+        Dir[@parse_dir + "*" + parse_suffix].each { |parsefilename|
+          filename_core = File.basename(parsefilename, parse_suffix)
+
+          # use iterator to read each parsed file
+          yield OneParsedFile.new(filename_core, parsefilename, sys)
+        }
+      end
+
+    else
+      # no parse:
+      # get pseudo-parse tree
+
+      if @stxml_dir
+        # use existing SalsaTigerXML files
+        Dir[@stxml_dir + "*.xml"].each { |stxmlfilename|
+
+          filename_core = File.basename(stxmlfilename, ".xml")
+          if @tab_dir
+            # we know the tab directory too
+            tabfilename = @tab_dir + filename_core + @file_suffixes["tab"]
+            each_sentence_obj = FrprepReadStxml.new(stxmlfilename, tabfilename,
+                                                    postag_suffix, lemma_suffix)
+          else
+            # we have no tab directory
+            each_sentence_obj = FrprepReadStxml.new(stxmlfilename, nil,
+                                                    postag_suffix, lemma_suffix)
+          end
+
+          yield OneParsedFile.new(filename_core, stxmlfilename, each_sentence_obj)
+        }
+
+      else
+        # construct SalsaTigerXML from tab files
+        Dir[@tab_dir+"*"+@file_suffixes["tab"]].each { |tabfilename|
+          each_sentence_obj = FrprepFlatSyntax.new(tabfilename,
+                                                   postag_suffix, 
+                                                   lemma_suffix)
+          filename_core = File.basename(tabfilename, @file_suffixes["tab"])
+          yield OneParsedFile.new(filename_core, tabfilename, each_sentence_obj)
+        }
+      end # source of pseudo-parse
+    end # parse or no parse
+  end 
+end