RubyGems - frprep - Versions diffs - 0.0.1.prealpha - Mend

frprep 0.0.1.prealpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (138) hide show

data/.yardopts +8 -0
data/CHANGELOG.rdoc +0 -0
data/LICENSE.rdoc +0 -0
data/README.rdoc +0 -0
data/lib/common/AbstractSynInterface.rb +1227 -0
data/lib/common/BerkeleyInterface.rb +375 -0
data/lib/common/CollinsInterface.rb +1165 -0
data/lib/common/ConfigData.rb +694 -0
data/lib/common/Counter.rb +18 -0
data/lib/common/DBInterface.rb +48 -0
data/lib/common/EnduserMode.rb +27 -0
data/lib/common/Eval.rb +480 -0
data/lib/common/FixSynSemMapping.rb +196 -0
data/lib/common/FrPrepConfigData.rb +66 -0
data/lib/common/FrprepHelper.rb +1324 -0
data/lib/common/Graph.rb +345 -0
data/lib/common/ISO-8859-1.rb +24 -0
data/lib/common/ML.rb +186 -0
data/lib/common/Maxent.rb +215 -0
data/lib/common/MiniparInterface.rb +1388 -0
data/lib/common/Optimise.rb +195 -0
data/lib/common/Parser.rb +213 -0
data/lib/common/RegXML.rb +269 -0
data/lib/common/RosyConventions.rb +171 -0
data/lib/common/SQLQuery.rb +243 -0
data/lib/common/STXmlTerminalOrder.rb +194 -0
data/lib/common/SalsaTigerRegXML.rb +2347 -0
data/lib/common/SalsaTigerXMLHelper.rb +99 -0
data/lib/common/SleepyInterface.rb +384 -0
data/lib/common/SynInterfaces.rb +275 -0
data/lib/common/TabFormat.rb +720 -0
data/lib/common/Tiger.rb +1448 -0
data/lib/common/TntInterface.rb +44 -0
data/lib/common/Tree.rb +61 -0
data/lib/common/TreetaggerInterface.rb +303 -0
data/lib/common/headz.rb +338 -0
data/lib/common/option_parser.rb +13 -0
data/lib/common/ruby_class_extensions.rb +310 -0
data/lib/fred/Baseline.rb +150 -0
data/lib/fred/FileZipped.rb +31 -0
data/lib/fred/FredBOWContext.rb +863 -0
data/lib/fred/FredConfigData.rb +182 -0
data/lib/fred/FredConventions.rb +232 -0
data/lib/fred/FredDetermineTargets.rb +324 -0
data/lib/fred/FredEval.rb +312 -0
data/lib/fred/FredFeatureExtractors.rb +321 -0
data/lib/fred/FredFeatures.rb +1061 -0
data/lib/fred/FredFeaturize.rb +596 -0
data/lib/fred/FredNumTrainingSenses.rb +27 -0
data/lib/fred/FredParameters.rb +402 -0
data/lib/fred/FredSplit.rb +84 -0
data/lib/fred/FredSplitPkg.rb +180 -0
data/lib/fred/FredTest.rb +607 -0
data/lib/fred/FredTrain.rb +144 -0
data/lib/fred/PlotAndREval.rb +480 -0
data/lib/fred/fred.rb +45 -0
data/lib/fred/md5.rb +23 -0
data/lib/fred/opt_parser.rb +250 -0
data/lib/frprep/AbstractSynInterface.rb +1227 -0
data/lib/frprep/Ampersand.rb +37 -0
data/lib/frprep/BerkeleyInterface.rb +375 -0
data/lib/frprep/CollinsInterface.rb +1165 -0
data/lib/frprep/ConfigData.rb +694 -0
data/lib/frprep/Counter.rb +18 -0
data/lib/frprep/FNCorpusXML.rb +643 -0
data/lib/frprep/FNDatabase.rb +144 -0
data/lib/frprep/FixSynSemMapping.rb +196 -0
data/lib/frprep/FrPrepConfigData.rb +66 -0
data/lib/frprep/FrameXML.rb +513 -0
data/lib/frprep/FrprepHelper.rb +1324 -0
data/lib/frprep/Graph.rb +345 -0
data/lib/frprep/ISO-8859-1.rb +24 -0
data/lib/frprep/MiniparInterface.rb +1388 -0
data/lib/frprep/Parser.rb +213 -0
data/lib/frprep/RegXML.rb +269 -0
data/lib/frprep/STXmlTerminalOrder.rb +194 -0
data/lib/frprep/SalsaTigerRegXML.rb +2347 -0
data/lib/frprep/SalsaTigerXMLHelper.rb +99 -0
data/lib/frprep/SleepyInterface.rb +384 -0
data/lib/frprep/SynInterfaces.rb +275 -0
data/lib/frprep/TabFormat.rb +720 -0
data/lib/frprep/Tiger.rb +1448 -0
data/lib/frprep/TntInterface.rb +44 -0
data/lib/frprep/Tree.rb +61 -0
data/lib/frprep/TreetaggerInterface.rb +303 -0
data/lib/frprep/do_parses.rb +142 -0
data/lib/frprep/frprep.rb +686 -0
data/lib/frprep/headz.rb +338 -0
data/lib/frprep/one_parsed_file.rb +28 -0
data/lib/frprep/opt_parser.rb +94 -0
data/lib/frprep/ruby_class_extensions.rb +310 -0
data/lib/rosy/AbstractFeatureAndExternal.rb +240 -0
data/lib/rosy/DBMySQL.rb +146 -0
data/lib/rosy/DBSQLite.rb +280 -0
data/lib/rosy/DBTable.rb +239 -0
data/lib/rosy/DBWrapper.rb +176 -0
data/lib/rosy/ExternalConfigData.rb +58 -0
data/lib/rosy/FailedParses.rb +130 -0
data/lib/rosy/FeatureInfo.rb +242 -0
data/lib/rosy/GfInduce.rb +1115 -0
data/lib/rosy/GfInduceFeature.rb +148 -0
data/lib/rosy/InputData.rb +294 -0
data/lib/rosy/RosyConfigData.rb +115 -0
data/lib/rosy/RosyConfusability.rb +338 -0
data/lib/rosy/RosyEval.rb +465 -0
data/lib/rosy/RosyFeatureExtractors.rb +1609 -0
data/lib/rosy/RosyFeaturize.rb +280 -0
data/lib/rosy/RosyInspect.rb +336 -0
data/lib/rosy/RosyIterator.rb +477 -0
data/lib/rosy/RosyPhase2FeatureExtractors.rb +230 -0
data/lib/rosy/RosyPruning.rb +165 -0
data/lib/rosy/RosyServices.rb +744 -0
data/lib/rosy/RosySplit.rb +232 -0
data/lib/rosy/RosyTask.rb +19 -0
data/lib/rosy/RosyTest.rb +826 -0
data/lib/rosy/RosyTrain.rb +232 -0
data/lib/rosy/RosyTrainingTestTable.rb +786 -0
data/lib/rosy/TargetsMostFrequentFrame.rb +60 -0
data/lib/rosy/View.rb +418 -0
data/lib/rosy/opt_parser.rb +379 -0
data/lib/rosy/rosy.rb +77 -0
data/lib/shalmaneser/version.rb +3 -0
data/test/frprep/test_opt_parser.rb +94 -0
data/test/functional/functional_test_helper.rb +40 -0
data/test/functional/sample_experiment_files/fred_test.salsa.erb +122 -0
data/test/functional/sample_experiment_files/fred_train.salsa.erb +135 -0
data/test/functional/sample_experiment_files/prp_test.salsa.erb +138 -0
data/test/functional/sample_experiment_files/prp_test.salsa.fred.standalone.erb +120 -0
data/test/functional/sample_experiment_files/prp_test.salsa.rosy.standalone.erb +120 -0
data/test/functional/sample_experiment_files/prp_train.salsa.erb +138 -0
data/test/functional/sample_experiment_files/prp_train.salsa.fred.standalone.erb +138 -0
data/test/functional/sample_experiment_files/prp_train.salsa.rosy.standalone.erb +138 -0
data/test/functional/sample_experiment_files/rosy_test.salsa.erb +257 -0
data/test/functional/sample_experiment_files/rosy_train.salsa.erb +259 -0
data/test/functional/test_fred.rb +47 -0
data/test/functional/test_frprep.rb +52 -0
data/test/functional/test_rosy.rb +20 -0
metadata +270 -0

data/lib/common/SalsaTigerXMLHelper.rb ADDED

@@ -0,0 +1,99 @@
+# sp jul 05 05
+#
+# Static helper methods for SalsaTigerRegXML:
+# - provide header and footer for Salsa/Tiger XML files
+# - escape and unescape HTML entities
+#
+# changed KE nov 05:
+# many methods moved to FrprepHelper
+require "common/SalsaTigerRegXML"
+require "common/headz"
+require "common/Parser"
+require "tempfile"
+class SalsaTigerXMLHelper
+  ###
+  # get header of SalsaTigerXML files (as string)
+  def SalsaTigerXMLHelper.get_header
+    header = <<ENDOFHEADER
+<?xml version="1.0" encoding="UTF-8"?>
+  <corpus corpusname="corpus" target="">
+	<head>
+		<meta>
+			<format>
+			NeGra format, version 3</format>
+		</meta>
+		<frames xmlns="http://www.clt-st.de/framenet/frame-database">
+		</frames>
+		<wordtags xmlns="http://www.clt-st.de/salsa/wordtags">
+		</wordtags>
+		<flags>
+		</flags>
+		<annotation>
+			<edgelabel>
+			</edgelabel>
+			<secedgelabel>
+			</secedgelabel>
+		</annotation>
+	</head>
+	<body>
+ENDOFHEADER
+    return header
+  end
+  ###
+  # get footer of SALSATigerXML files (as string)
+  def SalsaTigerXMLHelper.get_footer
+    footer = <<ENDOFFOOTER
+	</body>
+</corpus>
+ENDOFFOOTER
+    return footer
+  end
+# escape and unescape strings for representation in XML
+  @@replacements = [
+#  ["&apos;&apos;","&quot;"], # added by ines (09/03/09), might cause problems for unescape???
+  ["&","&amp;"], # must be first for escaping, last for unescaping
+  ["<","&lt;"],
+  [">", "&gt;"],
+  ["\"","&apos;&apos;"],
+#  ["\"","&quot;"],
+#  ["\'\'","&quot;"],
+#  ["\`\`","&quot;"],
+  ["\'","&apos;"],
+  ["\`\`","&apos;&apos;"],
+#  ["''","&apos;&apos;"]
+  ]
+def SalsaTigerXMLHelper.escape(string)
+  @@replacements.each {|unescaped,escaped|
+    string.gsub!(unescaped,escaped)
+  }
+  return string
+end
+def SalsaTigerXMLHelper.unescape(string)
+  # reverse replacements to replace &amp last
+  @@replacements.reverse.each {|unescaped,escaped|
+    string.gsub!(escaped,unescaped)
+  }
+  return string
+end
+end

data/lib/common/SleepyInterface.rb ADDED

@@ -0,0 +1,384 @@
+####
+# sp 21 07 05
+#
+# modified ke 30 10 05: adapted to fit into SynInterface
+#
+# represents a file containing Sleepy parses
+#
+# underlying data structure for individual sentences: SalsaTigerSentence
+require "tempfile"
+require "common/SalsaTigerRegXML"
+require "common/SalsaTigerXMLHelper"
+require "common/TabFormat"
+require "common/Counter"
+require "common/AbstractSynInterface"
+require "common/Tiger.rb"
+################################################
+# Interface class
+class SleepyInterface < SynInterfaceSTXML
+  SleepyInterface.announce_me()
+  ###
+  def SleepyInterface.system()
+    return "sleepy"
+  end
+  ###
+  def SleepyInterface.service()
+    return "parser"
+  end
+  ###
+  # initialize to set values for all subsequent processing
+  def initialize(program_path, # string: path to system
+		 insuffix,      # string: suffix of tab files
+		 outsuffix,     # string: suffix for parsed files
+		 stsuffix,      # string: suffix for Salsa/TIGER XML files
+		 var_hash = {}) # optional arguments in a hash
+    super(program_path, insuffix, outsuffix, stsuffix, var_hash)
+    unless @program_path =~ /\/$/
+      @program_path = @program_path + "/"
+    end
+    # new: evaluate var hash
+    @pos_suffix = var_hash["pos_suffix"]
+    @lemma_suffix = var_hash["lemma_suffix"]
+    @tab_dir = var_hash["tab_dir"]
+  end
+  ####
+  # parse a directory with TabFormat files and write the parse trees to outputdir
+  # I assume that the files in inputdir are smaller than
+  # the maximum number of sentences that
+  # Sleepy can parse in one go (i.e. that they are split)
+  def process_dir(in_dir,  # string: input directory name
+		  out_dir) # string: output directory name
+    sleepy_prog = "#{@program_path}sleepy  --beam 1000 --model-file #{@program_path}negra.model --parse "
+    Dir[in_dir + "*" + @insuffix].each {|inputfilename|
+      STDERR.puts "*** Parsing #{inputfilename} with Sleepy"
+      corpusfilename = File.basename(inputfilename, @insuffix)
+      parsefilename = out_dir + corpusfilename + @outsuffix
+      tempfile = Tempfile.new(corpusfilename)
+      # we need neither lemmata nor POS tags; sleepy can do with the words
+      corpusfile = FNTabFormatFile.new(inputfilename,nil, nil)
+      corpusfile.each_sentence {|sentence|
+        tempfile.puts sentence.to_s
+      }
+      tempfile.close
+      # parse and remove comments in the parser output
+      Kernel.system(sleepy_prog+" "+tempfile.path+" 2>&1 | grep -v \"Span:\" > "+parsefilename)
+    }
+  end
+  ###
+  # for a given parsed file:
+  # yield each sentence as a pair
+  #  [SalsaTigerSentence object, FNTabFormatSentence object]
+  # of the sentence in SalsaTigerXML and the matching tab format sentence
+  #
+  # If a parse has failed, returns
+  #  [failed_sentence (flat SalsaTigerSentence), FNTabFormatSentence]
+  # to allow more detailed accounting for failed parses
+  # (basically just a flat structure with a failed=true attribute
+  # at the sentence node)
+  def each_sentence(parsefilename)
+    # sanity checks
+    unless @tab_dir
+      $stderr.puts "SleepyInterface error: Need to set tab directory on initialization"
+      exit 1
+    end
+    # get matching tab file for this parser output file
+    parsefile = File.new(parsefilename)
+    tabfilename = @tab_dir+File.basename(parsefilename, @outsuffix)+ @insuffix
+    tabfile = FNTabFormatFile.new(tabfilename, @postag_suffix, @lemma_suffix)
+    sentid = 0
+    tabfile.each_sentence {|tab_sent| # iterate over corpus sentences
+      sentence_str = ""
+      status = true # error encountered?
+      # assemble next sentence in Sleepy file by reading lines from parsefile
+      while true
+        line = parsefile.gets
+        case line
+        when /% Parse failed/
+          status = false
+          break
+        when nil # end of file: nothing more to break
+          break
+        when /^%/, /^\s*$/ # empty lines, other comments: end of current sentence
+          unless sentence_str == "" # only break if you have read something
+            break
+          end
+        else
+          sentence_str += line.chomp # collect line of current parse and continue reading
+        end
+      end
+      # we have reached some kind of end
+      sentid +=1
+      # we don't have a sentence: hopefully, this is becase parsing has failed
+      # if this is not the case, we are in trouble
+      if sentence_str == ""
+        case status
+        when false
+          # return a SalsaTigerSentence object for the failed sentence
+          # with a virtual top node and one terminal per word.
+          if tab_sent.get_sent_id() and tab_sent.get_sent_id() != "--"
+            my_sent_id = tab_sent.get_sent_id()
+          else
+            my_sent_id = File.basename(parsefilename, @outsuffix) + "_" + sentid.to_s
+          end
+          sent = SleepyInterface.failed_sentence(tab_sent, my_sent_id)
+          yield [sent, tab_sent, SleepyInterface.standard_mapping(sent, tab_sent)]
+        else
+	  # this may not happen: we need some sentence for the current
+	  # TabFile sentence
+          $stderr.puts "SleepyInterface error: premature end of parser file!"
+          exit 1
+        end
+      else
+        # if we are here, we have a sentence_str to work on
+        # hopefully, our status is OK
+        case status
+        when true
+          if tab_sent.get_sent_id() and tab_sent.get_sent_id() != "--"
+            my_sent_id = tab_sent.get_sent_id()
+          else
+            my_sent_id = File.basename(parsefilename, @outsuffix) + "_" + sentid.to_s
+          end
+          st_sent = build_salsatiger(" " + sentence_str + " ", 0,
+				     Array.new, Counter.new(0),
+				     Counter.new(500),
+				     SalsaTigerSentence.empty_sentence(my_sent_id.to_s))
+          yield [st_sent, tab_sent, SleepyInterface.standard_mapping(st_sent, tab_sent)]
+        else # i.e. when "failed"
+          $stderr.puts "SleepyInterface error: failed parse, but parse tree exists??"
+          exit 1
+        end
+      end
+    }
+    # all TabFile sentences are consumed:
+    # now we may just encounter comments, garbage, empty lines etc.
+    while not parsefile.eof?
+      case parsefile.gets
+      when nil, /^%/, /^\s*$/ # empty lines, comments, end of input indicate end of current parse
+      else
+        $stderr.puts "SleepyInterface error: premature end of tab file"
+        exit 1
+      end
+    end
+  end
+  ###
+  # write Salsa/TIGER XML output to file
+  def to_stxml_file(infilename,  # string: name of parse file
+		    outfilename) # string: name of output stxml file
+    outfile = File.new(outfilename, "w")
+    outfile.puts SalsaTigerXMLHelper.get_header()
+    each_sentence(infilename) { |st_sent, tabsent|
+      outfile.puts st_sent.get()
+    }
+    outfile.puts SalsaTigerXMLHelper.get_footer()
+    outfile.close()
+  end
+  ########################
+  private
+  ###
+  # Recursive function for parsing a Sleepy parse tree and
+  # building a SalsaTigerSentence recursively
+  #
+  # Algorithm: manage stack which contains, for the current constituent,
+  # child constituents (if a nonterminal), and the category label.
+  # When the end of a constituent is reached, a new SynNode (TigerSalsa node) ist created.
+  # All children and the category label are popped from the stack and integrated into the
+  # TigerSalsa data structure. The new node is re-pushed onto the stack.
+  def build_salsatiger(sentence, # string
+                    pos,      # position in string (index): integer
+                    stack,    # stack with incomplete nodes: Array
+                    termc,    # terminal counter
+                    nontc,    # nonterminal counter
+                    sent_obj) # SalsaTigerSentence
+    # main case distinction: match the beginning of our string
+    # (i.e. what follows our current position in the string)
+    case sentence[pos..-1]
+    when /^ *$/ # nothing -> whole sentence parsed
+      if stack.length == 1
+	# sleepy always delivers one "top" node; if we don't get just one
+        # node, something has gone wrong
+        node = stack.pop
+        node.del_attribute("gf")
+        return sent_obj
+      else
+        $stderr.puts "SleepyINterface Error: more than one root node (stack length #{stack.length}). Full sentence: \n#{sentence}"
+        exit 1
+      end
+    when /^\s*\(([^ )]+) /
+      # match the beginning of a new constituent
+      # (opening bracket + category + space, may not contain closing bracket)
+      cat = $1
+      if cat.nil? or cat == ""
+        $stderr.puts "SleepyInterface Error: found category nil in sentence #{sentence[pos,10]}, full sentence\n#{sentence}"
+        exit 1
+      end
+#          STDERR.puts "new const #{cat}"
+      stack.push cat # throw the category label on the stack
+      return build_salsatiger(sentence,pos+$&.length,stack,termc,nontc,sent_obj)
+    when /^\s*(\S+)\) /
+      # match the end of a terminal constituent (something before a closing bracket + space)
+      word = $1
+      comb_cat = stack.pop
+      if comb_cat.to_s == ""
+        $stderr.puts "SleepyInterface error: Empty cat at position #{sentence[pos,10]}, full sentence\n#{sentence}"
+        exit 1
+      end
+      cat,gf = split_cat(comb_cat)
+      node = sent_obj.add_syn("t",
+                              nil,  # cat (doesn't matter here)
+                              SalsaTigerXMLHelper.escape(word), # word
+                              cat,  # pos
+                              termc.next.to_s)
+      node.set_attribute("gf",gf)
+#          STDERR.puts "completed terminal #{cat}, #{word}"
+      stack.push node
+      return build_salsatiger(sentence,pos+$&.length,stack,termc,nontc,sent_obj)
+    when /^\s*\)/ # match the end of a nonterminal (nothing before a closing bracket)
+      # now collect children:
+      # pop items from the stack until you find the category
+      children = Array.new
+      while true
+        if stack.empty?
+          $stderr.puts  "SleepyInterface Error: stack empty; cannot find more children"
+          exit 1
+        end
+        item = stack.pop
+        case item.class.to_s
+        when "SynNode" # this is a child
+          children.push item
+        when "String" # this is the category label
+          if item.to_s == ""
+            $stderr.puts "SleepyInterface error: Empty cat at position #{sentence[pos,10]}, full sentence\n#{sentence}"
+            exit 1
+          end
+          cat,gf = split_cat(item)
+          break
+        else
+          $stderr.puts "SleepyInterface Error: unknown item class #{item.class.to_s}"
+          exit 1
+        end
+      end
+      # now add a nonterminal node to the sentence object and
+      # register the children nodes
+      node = sent_obj.add_syn("nt",
+                              cat, # cat
+                              nil, # word (doesn't matter)
+                              nil, # pos (doesn't matter)
+                              nontc.next.to_s)
+      children.each {|child|
+        child_gf = child.get_attribute("gf")
+        child.del_attribute("gf")
+        node.add_child(child,child_gf)
+        child.add_parent(node, child_gf)
+      }
+      node.set_attribute("gf",gf)
+#          STDERR.puts "Completed nonterm #{cat}, #{children.length} children."
+      stack.push node
+      return build_salsatiger(sentence,pos+$&.length, stack,termc,nontc,sent_obj)
+    else
+      if sentence =~ /Fatal error: exception Out_of_memory/
+        $stderr.puts "SleepyInterface error: Sleepy parser ran out of memory."
+        $stderr.puts "Try reducing the max. sentence length"
+        $stderr.puts "in the experiment file."
+        exit 1
+      end
+      $stderr.puts "SleepyInterface Error: cannot analyse sentence at pos #{pos}:\n #{sentence[pos..-1]}\n Complete sentence: \n#{sentence}"
+      exit 1
+    end
+  end
+  ###
+  # Sleepy delivers node labels as "phrase type"-"grammatical function"
+  # but the GF may not be present.
+  def split_cat(cat)
+    cat =~ /^([^-]*)(-([^-]*))?$/
+    unless $1
+      $stderr.puts "SleepyInterface Error: could not identify category in #{cat}"
+      exit 1
+    end
+    proper_cat = $1
+    if $3
+      gf = $3
+    else
+      gf = ""
+    end
+    return [proper_cat,gf]
+  end
+end
+################################################
+# Interpreter class
+class SleepyInterpreter < Tiger
+  SleepyInterpreter.announce_me()
+  ###
+  # names of the systems interpreted by this class:
+  # returns a hash service(string) -> system name (string),
+  # e.g.
+  # { "parser" => "collins", "lemmatizer" => "treetagger" }
+  def SleepyInterpreter.systems()
+    return {
+	"parser" => "sleepy"
+    }
+  end
+  ###
+  # names of additional systems that may be interpreted by this class
+  # returns a hash service(string) -> system name(string)
+  # same as names()
+  def SleepyInterpreter.optional_systems()
+    return {
+      "lemmatizer" => "treetagger"
+    }
+  end
+end