RubyGems - shalmaneser - Versions diffs - 0.0.1.alpha - Mend

shalmaneser 0.0.1.alpha

Files changed (138) hide show

data/.yardopts +8 -0
data/CHANGELOG.rdoc +0 -0
data/LICENSE.rdoc +0 -0
data/README.rdoc +0 -0
data/lib/common/AbstractSynInterface.rb +1227 -0
data/lib/common/BerkeleyInterface.rb +375 -0
data/lib/common/CollinsInterface.rb +1165 -0
data/lib/common/ConfigData.rb +694 -0
data/lib/common/Counter.rb +18 -0
data/lib/common/DBInterface.rb +48 -0
data/lib/common/EnduserMode.rb +27 -0
data/lib/common/Eval.rb +480 -0
data/lib/common/FixSynSemMapping.rb +196 -0
data/lib/common/FrPrepConfigData.rb +66 -0
data/lib/common/FrprepHelper.rb +1324 -0
data/lib/common/Graph.rb +345 -0
data/lib/common/ISO-8859-1.rb +24 -0
data/lib/common/ML.rb +186 -0
data/lib/common/Maxent.rb +215 -0
data/lib/common/MiniparInterface.rb +1388 -0
data/lib/common/Optimise.rb +195 -0
data/lib/common/Parser.rb +213 -0
data/lib/common/RegXML.rb +269 -0
data/lib/common/RosyConventions.rb +171 -0
data/lib/common/SQLQuery.rb +243 -0
data/lib/common/STXmlTerminalOrder.rb +194 -0
data/lib/common/SalsaTigerRegXML.rb +2347 -0
data/lib/common/SalsaTigerXMLHelper.rb +99 -0
data/lib/common/SleepyInterface.rb +384 -0
data/lib/common/SynInterfaces.rb +275 -0
data/lib/common/TabFormat.rb +720 -0
data/lib/common/Tiger.rb +1448 -0
data/lib/common/TntInterface.rb +44 -0
data/lib/common/Tree.rb +61 -0
data/lib/common/TreetaggerInterface.rb +303 -0
data/lib/common/headz.rb +338 -0
data/lib/common/option_parser.rb +13 -0
data/lib/common/ruby_class_extensions.rb +310 -0
data/lib/fred/Baseline.rb +150 -0
data/lib/fred/FileZipped.rb +31 -0
data/lib/fred/FredBOWContext.rb +863 -0
data/lib/fred/FredConfigData.rb +182 -0
data/lib/fred/FredConventions.rb +232 -0
data/lib/fred/FredDetermineTargets.rb +324 -0
data/lib/fred/FredEval.rb +312 -0
data/lib/fred/FredFeatureExtractors.rb +321 -0
data/lib/fred/FredFeatures.rb +1061 -0
data/lib/fred/FredFeaturize.rb +596 -0
data/lib/fred/FredNumTrainingSenses.rb +27 -0
data/lib/fred/FredParameters.rb +402 -0
data/lib/fred/FredSplit.rb +84 -0
data/lib/fred/FredSplitPkg.rb +180 -0
data/lib/fred/FredTest.rb +607 -0
data/lib/fred/FredTrain.rb +144 -0
data/lib/fred/PlotAndREval.rb +480 -0
data/lib/fred/fred.rb +45 -0
data/lib/fred/md5.rb +23 -0
data/lib/fred/opt_parser.rb +250 -0
data/lib/frprep/AbstractSynInterface.rb +1227 -0
data/lib/frprep/Ampersand.rb +37 -0
data/lib/frprep/BerkeleyInterface.rb +375 -0
data/lib/frprep/CollinsInterface.rb +1165 -0
data/lib/frprep/ConfigData.rb +694 -0
data/lib/frprep/Counter.rb +18 -0
data/lib/frprep/FNCorpusXML.rb +643 -0
data/lib/frprep/FNDatabase.rb +144 -0
data/lib/frprep/FixSynSemMapping.rb +196 -0
data/lib/frprep/FrPrepConfigData.rb +66 -0
data/lib/frprep/FrameXML.rb +513 -0
data/lib/frprep/FrprepHelper.rb +1324 -0
data/lib/frprep/Graph.rb +345 -0
data/lib/frprep/ISO-8859-1.rb +24 -0
data/lib/frprep/MiniparInterface.rb +1388 -0
data/lib/frprep/Parser.rb +213 -0
data/lib/frprep/RegXML.rb +269 -0
data/lib/frprep/STXmlTerminalOrder.rb +194 -0
data/lib/frprep/SalsaTigerRegXML.rb +2347 -0
data/lib/frprep/SalsaTigerXMLHelper.rb +99 -0
data/lib/frprep/SleepyInterface.rb +384 -0
data/lib/frprep/SynInterfaces.rb +275 -0
data/lib/frprep/TabFormat.rb +720 -0
data/lib/frprep/Tiger.rb +1448 -0
data/lib/frprep/TntInterface.rb +44 -0
data/lib/frprep/Tree.rb +61 -0
data/lib/frprep/TreetaggerInterface.rb +303 -0
data/lib/frprep/do_parses.rb +142 -0
data/lib/frprep/frprep.rb +686 -0
data/lib/frprep/headz.rb +338 -0
data/lib/frprep/one_parsed_file.rb +28 -0
data/lib/frprep/opt_parser.rb +94 -0
data/lib/frprep/ruby_class_extensions.rb +310 -0
data/lib/rosy/AbstractFeatureAndExternal.rb +240 -0
data/lib/rosy/DBMySQL.rb +146 -0
data/lib/rosy/DBSQLite.rb +280 -0
data/lib/rosy/DBTable.rb +239 -0
data/lib/rosy/DBWrapper.rb +176 -0
data/lib/rosy/ExternalConfigData.rb +58 -0
data/lib/rosy/FailedParses.rb +130 -0
data/lib/rosy/FeatureInfo.rb +242 -0
data/lib/rosy/GfInduce.rb +1115 -0
data/lib/rosy/GfInduceFeature.rb +148 -0
data/lib/rosy/InputData.rb +294 -0
data/lib/rosy/RosyConfigData.rb +115 -0
data/lib/rosy/RosyConfusability.rb +338 -0
data/lib/rosy/RosyEval.rb +465 -0
data/lib/rosy/RosyFeatureExtractors.rb +1609 -0
data/lib/rosy/RosyFeaturize.rb +280 -0
data/lib/rosy/RosyInspect.rb +336 -0
data/lib/rosy/RosyIterator.rb +477 -0
data/lib/rosy/RosyPhase2FeatureExtractors.rb +230 -0
data/lib/rosy/RosyPruning.rb +165 -0
data/lib/rosy/RosyServices.rb +744 -0
data/lib/rosy/RosySplit.rb +232 -0
data/lib/rosy/RosyTask.rb +19 -0
data/lib/rosy/RosyTest.rb +826 -0
data/lib/rosy/RosyTrain.rb +232 -0
data/lib/rosy/RosyTrainingTestTable.rb +786 -0
data/lib/rosy/TargetsMostFrequentFrame.rb +60 -0
data/lib/rosy/View.rb +418 -0
data/lib/rosy/opt_parser.rb +379 -0
data/lib/rosy/rosy.rb +77 -0
data/lib/shalmaneser/version.rb +3 -0
data/test/frprep/test_opt_parser.rb +94 -0
data/test/functional/functional_test_helper.rb +40 -0
data/test/functional/sample_experiment_files/fred_test.salsa.erb +122 -0
data/test/functional/sample_experiment_files/fred_train.salsa.erb +135 -0
data/test/functional/sample_experiment_files/prp_test.salsa.erb +138 -0
data/test/functional/sample_experiment_files/prp_test.salsa.fred.standalone.erb +120 -0
data/test/functional/sample_experiment_files/prp_test.salsa.rosy.standalone.erb +120 -0
data/test/functional/sample_experiment_files/prp_train.salsa.erb +138 -0
data/test/functional/sample_experiment_files/prp_train.salsa.fred.standalone.erb +138 -0
data/test/functional/sample_experiment_files/prp_train.salsa.rosy.standalone.erb +138 -0
data/test/functional/sample_experiment_files/rosy_test.salsa.erb +257 -0
data/test/functional/sample_experiment_files/rosy_train.salsa.erb +259 -0
data/test/functional/test_fred.rb +47 -0
data/test/functional/test_frprep.rb +52 -0
data/test/functional/test_rosy.rb +20 -0
metadata +284 -0

data/lib/common/SalsaTigerXMLHelper.rb ADDED Viewed

@@ -0,0 +1,99 @@
+# sp jul 05 05
+#
+# Static helper methods for SalsaTigerRegXML:
+# - provide header and footer for Salsa/Tiger XML files
+# - escape and unescape HTML entities
+#
+# changed KE nov 05:
+# many methods moved to FrprepHelper
+require "common/SalsaTigerRegXML"
+require "common/headz"
+require "common/Parser"
+require "tempfile"
+class SalsaTigerXMLHelper
+  ###
+  # get header of SalsaTigerXML files (as string)
+  def SalsaTigerXMLHelper.get_header
+    header = <<ENDOFHEADER
+<?xml version="1.0" encoding="UTF-8"?>
+  <corpus corpusname="corpus" target="">
+	<head>
+		<meta>
+			<format>
+			NeGra format, version 3</format>
+		</meta>
+		<frames xmlns="http://www.clt-st.de/framenet/frame-database">
+		</frames>
+		<wordtags xmlns="http://www.clt-st.de/salsa/wordtags">
+		</wordtags>
+		<flags>
+		</flags>
+		<annotation>
+			<edgelabel>
+			</edgelabel>
+			<secedgelabel>
+			</secedgelabel>
+		</annotation>
+	</head>
+	<body>
+ENDOFHEADER
+    return header
+  end
+  ###
+  # get footer of SALSATigerXML files (as string)
+  def SalsaTigerXMLHelper.get_footer
+    footer = <<ENDOFFOOTER
+	</body>
+</corpus>
+ENDOFFOOTER
+    return footer
+  end
+# escape and unescape strings for representation in XML
+  @@replacements = [
+#  ["&apos;&apos;","&quot;"], # added by ines (09/03/09), might cause problems for unescape???
+  ["&","&amp;"], # must be first for escaping, last for unescaping
+  ["<","&lt;"],
+  [">", "&gt;"],
+  ["\"","&apos;&apos;"],
+#  ["\"","&quot;"],
+#  ["\'\'","&quot;"],
+#  ["\`\`","&quot;"],
+  ["\'","&apos;"],
+  ["\`\`","&apos;&apos;"],
+#  ["''","&apos;&apos;"]
+  ]
+def SalsaTigerXMLHelper.escape(string)
+  @@replacements.each {|unescaped,escaped|
+    string.gsub!(unescaped,escaped)
+  }
+  return string
+end
+def SalsaTigerXMLHelper.unescape(string)
+  # reverse replacements to replace &amp last
+  @@replacements.reverse.each {|unescaped,escaped|
+    string.gsub!(escaped,unescaped)
+  }
+  return string
+end
+end

data/lib/common/SleepyInterface.rb ADDED Viewed

@@ -0,0 +1,384 @@
+####
+# sp 21 07 05
+#
+# modified ke 30 10 05: adapted to fit into SynInterface
+#
+# represents a file containing Sleepy parses
+#
+# underlying data structure for individual sentences: SalsaTigerSentence
+require "tempfile"
+require "common/SalsaTigerRegXML"
+require "common/SalsaTigerXMLHelper"
+require "common/TabFormat"
+require "common/Counter"
+require "common/AbstractSynInterface"
+require "common/Tiger.rb"
+################################################
+# Interface class
+class SleepyInterface < SynInterfaceSTXML
+  SleepyInterface.announce_me()
+  ###
+  def SleepyInterface.system()
+    return "sleepy"
+  end
+  ###
+  def SleepyInterface.service()
+    return "parser"
+  end
+  ###
+  # initialize to set values for all subsequent processing
+  def initialize(program_path, # string: path to system
+		 insuffix,      # string: suffix of tab files
+		 outsuffix,     # string: suffix for parsed files
+		 stsuffix,      # string: suffix for Salsa/TIGER XML files
+		 var_hash = {}) # optional arguments in a hash
+    super(program_path, insuffix, outsuffix, stsuffix, var_hash)
+    unless @program_path =~ /\/$/
+      @program_path = @program_path + "/"
+    end
+    # new: evaluate var hash
+    @pos_suffix = var_hash["pos_suffix"]
+    @lemma_suffix = var_hash["lemma_suffix"]
+    @tab_dir = var_hash["tab_dir"]
+  end
+  ####
+  # parse a directory with TabFormat files and write the parse trees to outputdir
+  # I assume that the files in inputdir are smaller than
+  # the maximum number of sentences that
+  # Sleepy can parse in one go (i.e. that they are split)
+  def process_dir(in_dir,  # string: input directory name
+		  out_dir) # string: output directory name
+    sleepy_prog = "#{@program_path}sleepy  --beam 1000 --model-file #{@program_path}negra.model --parse "
+    Dir[in_dir + "*" + @insuffix].each {|inputfilename|
+      STDERR.puts "*** Parsing #{inputfilename} with Sleepy"
+      corpusfilename = File.basename(inputfilename, @insuffix)
+      parsefilename = out_dir + corpusfilename + @outsuffix
+      tempfile = Tempfile.new(corpusfilename)
+      # we need neither lemmata nor POS tags; sleepy can do with the words
+      corpusfile = FNTabFormatFile.new(inputfilename,nil, nil)
+      corpusfile.each_sentence {|sentence|
+        tempfile.puts sentence.to_s
+      }
+      tempfile.close
+      # parse and remove comments in the parser output
+      Kernel.system(sleepy_prog+" "+tempfile.path+" 2>&1 | grep -v \"Span:\" > "+parsefilename)
+    }
+  end
+  ###
+  # for a given parsed file:
+  # yield each sentence as a pair
+  #  [SalsaTigerSentence object, FNTabFormatSentence object]
+  # of the sentence in SalsaTigerXML and the matching tab format sentence
+  #
+  # If a parse has failed, returns
+  #  [failed_sentence (flat SalsaTigerSentence), FNTabFormatSentence]
+  # to allow more detailed accounting for failed parses
+  # (basically just a flat structure with a failed=true attribute
+  # at the sentence node)
+  def each_sentence(parsefilename)
+    # sanity checks
+    unless @tab_dir
+      $stderr.puts "SleepyInterface error: Need to set tab directory on initialization"
+      exit 1
+    end
+    # get matching tab file for this parser output file
+    parsefile = File.new(parsefilename)
+    tabfilename = @tab_dir+File.basename(parsefilename, @outsuffix)+ @insuffix
+    tabfile = FNTabFormatFile.new(tabfilename, @postag_suffix, @lemma_suffix)
+    sentid = 0
+    tabfile.each_sentence {|tab_sent| # iterate over corpus sentences
+      sentence_str = ""
+      status = true # error encountered?
+      # assemble next sentence in Sleepy file by reading lines from parsefile
+      while true
+        line = parsefile.gets
+        case line
+        when /% Parse failed/
+          status = false
+          break
+        when nil # end of file: nothing more to break
+          break
+        when /^%/, /^\s*$/ # empty lines, other comments: end of current sentence
+          unless sentence_str == "" # only break if you have read something
+            break
+          end
+        else
+          sentence_str += line.chomp # collect line of current parse and continue reading
+        end
+      end
+      # we have reached some kind of end
+      sentid +=1
+      # we don't have a sentence: hopefully, this is becase parsing has failed
+      # if this is not the case, we are in trouble
+      if sentence_str == ""
+        case status
+        when false
+          # return a SalsaTigerSentence object for the failed sentence
+          # with a virtual top node and one terminal per word.
+          if tab_sent.get_sent_id() and tab_sent.get_sent_id() != "--"
+            my_sent_id = tab_sent.get_sent_id()
+          else
+            my_sent_id = File.basename(parsefilename, @outsuffix) + "_" + sentid.to_s
+          end
+          sent = SleepyInterface.failed_sentence(tab_sent, my_sent_id)
+          yield [sent, tab_sent, SleepyInterface.standard_mapping(sent, tab_sent)]
+        else
+	  # this may not happen: we need some sentence for the current
+	  # TabFile sentence
+          $stderr.puts "SleepyInterface error: premature end of parser file!"
+          exit 1
+        end
+      else
+        # if we are here, we have a sentence_str to work on
+        # hopefully, our status is OK
+        case status
+        when true
+          if tab_sent.get_sent_id() and tab_sent.get_sent_id() != "--"
+            my_sent_id = tab_sent.get_sent_id()
+          else
+            my_sent_id = File.basename(parsefilename, @outsuffix) + "_" + sentid.to_s
+          end
+          st_sent = build_salsatiger(" " + sentence_str + " ", 0,
+				     Array.new, Counter.new(0),
+				     Counter.new(500),
+				     SalsaTigerSentence.empty_sentence(my_sent_id.to_s))
+          yield [st_sent, tab_sent, SleepyInterface.standard_mapping(st_sent, tab_sent)]
+        else # i.e. when "failed"
+          $stderr.puts "SleepyInterface error: failed parse, but parse tree exists??"
+          exit 1
+        end
+      end
+    }
+    # all TabFile sentences are consumed:
+    # now we may just encounter comments, garbage, empty lines etc.
+    while not parsefile.eof?
+      case parsefile.gets
+      when nil, /^%/, /^\s*$/ # empty lines, comments, end of input indicate end of current parse
+      else
+        $stderr.puts "SleepyInterface error: premature end of tab file"
+        exit 1
+      end
+    end
+  end
+  ###
+  # write Salsa/TIGER XML output to file
+  def to_stxml_file(infilename,  # string: name of parse file
+		    outfilename) # string: name of output stxml file
+    outfile = File.new(outfilename, "w")
+    outfile.puts SalsaTigerXMLHelper.get_header()
+    each_sentence(infilename) { |st_sent, tabsent|
+      outfile.puts st_sent.get()
+    }
+    outfile.puts SalsaTigerXMLHelper.get_footer()
+    outfile.close()
+  end
+  ########################
+  private
+  ###
+  # Recursive function for parsing a Sleepy parse tree and
+  # building a SalsaTigerSentence recursively
+  #
+  # Algorithm: manage stack which contains, for the current constituent,
+  # child constituents (if a nonterminal), and the category label.
+  # When the end of a constituent is reached, a new SynNode (TigerSalsa node) ist created.
+  # All children and the category label are popped from the stack and integrated into the
+  # TigerSalsa data structure. The new node is re-pushed onto the stack.
+  def build_salsatiger(sentence, # string
+                    pos,      # position in string (index): integer
+                    stack,    # stack with incomplete nodes: Array
+                    termc,    # terminal counter
+                    nontc,    # nonterminal counter
+                    sent_obj) # SalsaTigerSentence
+    # main case distinction: match the beginning of our string
+    # (i.e. what follows our current position in the string)
+    case sentence[pos..-1]
+    when /^ *$/ # nothing -> whole sentence parsed
+      if stack.length == 1
+	# sleepy always delivers one "top" node; if we don't get just one
+        # node, something has gone wrong
+        node = stack.pop
+        node.del_attribute("gf")
+        return sent_obj
+      else
+        $stderr.puts "SleepyINterface Error: more than one root node (stack length #{stack.length}). Full sentence: \n#{sentence}"
+        exit 1
+      end
+    when /^\s*\(([^ )]+) /
+      # match the beginning of a new constituent
+      # (opening bracket + category + space, may not contain closing bracket)
+      cat = $1
+      if cat.nil? or cat == ""
+        $stderr.puts "SleepyInterface Error: found category nil in sentence #{sentence[pos,10]}, full sentence\n#{sentence}"
+        exit 1
+      end
+#          STDERR.puts "new const #{cat}"
+      stack.push cat # throw the category label on the stack
+      return build_salsatiger(sentence,pos+$&.length,stack,termc,nontc,sent_obj)
+    when /^\s*(\S+)\) /
+      # match the end of a terminal constituent (something before a closing bracket + space)
+      word = $1
+      comb_cat = stack.pop
+      if comb_cat.to_s == ""
+        $stderr.puts "SleepyInterface error: Empty cat at position #{sentence[pos,10]}, full sentence\n#{sentence}"
+        exit 1
+      end
+      cat,gf = split_cat(comb_cat)
+      node = sent_obj.add_syn("t",
+                              nil,  # cat (doesn't matter here)
+                              SalsaTigerXMLHelper.escape(word), # word
+                              cat,  # pos
+                              termc.next.to_s)
+      node.set_attribute("gf",gf)
+#          STDERR.puts "completed terminal #{cat}, #{word}"
+      stack.push node
+      return build_salsatiger(sentence,pos+$&.length,stack,termc,nontc,sent_obj)
+    when /^\s*\)/ # match the end of a nonterminal (nothing before a closing bracket)
+      # now collect children:
+      # pop items from the stack until you find the category
+      children = Array.new
+      while true
+        if stack.empty?
+          $stderr.puts  "SleepyInterface Error: stack empty; cannot find more children"
+          exit 1
+        end
+        item = stack.pop
+        case item.class.to_s
+        when "SynNode" # this is a child
+          children.push item
+        when "String" # this is the category label
+          if item.to_s == ""
+            $stderr.puts "SleepyInterface error: Empty cat at position #{sentence[pos,10]}, full sentence\n#{sentence}"
+            exit 1
+          end
+          cat,gf = split_cat(item)
+          break
+        else
+          $stderr.puts "SleepyInterface Error: unknown item class #{item.class.to_s}"
+          exit 1
+        end
+      end
+      # now add a nonterminal node to the sentence object and
+      # register the children nodes
+      node = sent_obj.add_syn("nt",
+                              cat, # cat
+                              nil, # word (doesn't matter)
+                              nil, # pos (doesn't matter)
+                              nontc.next.to_s)
+      children.each {|child|
+        child_gf = child.get_attribute("gf")
+        child.del_attribute("gf")
+        node.add_child(child,child_gf)
+        child.add_parent(node, child_gf)
+      }
+      node.set_attribute("gf",gf)
+#          STDERR.puts "Completed nonterm #{cat}, #{children.length} children."
+      stack.push node
+      return build_salsatiger(sentence,pos+$&.length, stack,termc,nontc,sent_obj)
+    else
+      if sentence =~ /Fatal error: exception Out_of_memory/
+        $stderr.puts "SleepyInterface error: Sleepy parser ran out of memory."
+        $stderr.puts "Try reducing the max. sentence length"
+        $stderr.puts "in the experiment file."
+        exit 1
+      end
+      $stderr.puts "SleepyInterface Error: cannot analyse sentence at pos #{pos}:\n #{sentence[pos..-1]}\n Complete sentence: \n#{sentence}"
+      exit 1
+    end
+  end
+  ###
+  # Sleepy delivers node labels as "phrase type"-"grammatical function"
+  # but the GF may not be present.
+  def split_cat(cat)
+    cat =~ /^([^-]*)(-([^-]*))?$/
+    unless $1
+      $stderr.puts "SleepyInterface Error: could not identify category in #{cat}"
+      exit 1
+    end
+    proper_cat = $1
+    if $3
+      gf = $3
+    else
+      gf = ""
+    end
+    return [proper_cat,gf]
+  end
+end
+################################################
+# Interpreter class
+class SleepyInterpreter < Tiger
+  SleepyInterpreter.announce_me()
+  ###
+  # names of the systems interpreted by this class:
+  # returns a hash service(string) -> system name (string),
+  # e.g.
+  # { "parser" => "collins", "lemmatizer" => "treetagger" }
+  def SleepyInterpreter.systems()
+    return {
+	"parser" => "sleepy"
+    }
+  end
+  ###
+  # names of additional systems that may be interpreted by this class
+  # returns a hash service(string) -> system name(string)
+  # same as names()
+  def SleepyInterpreter.optional_systems()
+    return {
+      "lemmatizer" => "treetagger"
+    }
+  end
+end