shalmaneser-frappe 1.2.rc5

data/lib/frappe/interfaces/collins_interface.rb

@@ -0,0 +1,340 @@
+require_relative 'counter'
+
+require 'frappe/syn_interface_stxml'
+require 'salsa_tiger_xml/salsa_tiger_sentence'
+require 'salsa_tiger_xml/syn_node'
+require 'tabular_format/fn_tab_format_file'
+
+require 'tempfile'
+
+# Interface class
+# p self.class.constants
+module Shalmaneser
+  # p self.constants
+  module Frappe
+    # p self.constants
+    class CollinsInterface < SynInterfaceSTXML
+      CollinsInterface.announce_me
+
+      ###
+      def self.system
+        "collins"
+      end
+
+      ###
+      def self.service
+        "parser"
+      end
+
+      ###
+      # initialize to set values for all subsequent processing
+      def initialize(program_path, # string: path to system
+                     insuffix,      # string: suffix of tab files
+                     outsuffix,     # string: suffix for parsed files
+                     stsuffix,      # string: suffix for Sals/TIGER XML files
+                     var_hash = {}) # optional arguments in a hash
+
+        super(program_path, insuffix, outsuffix, stsuffix, var_hash)
+        # I am not expecting any parameters, but I need
+        # the program path to end in a /.
+        unless @program_path =~ /\/$/
+          @program_path = @program_path + "/"
+        end
+
+        # new: evaluate var hash
+        @pos_suffix = var_hash["pos_suffix"]
+        @lemma_suffix = var_hash["lemma_suffix"]
+        @tab_dir = var_hash["tab_dir"]
+      end
+
+
+      ###
+      # parse a bunch of TabFormat files (*.<insuffix>) with Collins model 3
+      # required: POS tags must be present
+      # produced: in outputdir, files *.<outsuffix>
+      # I assume that the files in inputdir are smaller than
+      # the maximum number of sentences
+      # Collins can parse in one go (i.e. that they are split) and I don't have to care
+      def process_dir(in_dir,        # string: name of input directory
+                      out_dir)       # string: name of output directory
+        print "parsing ", in_dir, " and writing to ", out_dir, "\n"
+
+        unless @pos_suffix
+          raise "Collins interface: need suffix for POS files"
+        end
+
+        collins_prog = "gunzip -c #{@program_path}models/model3/events.gz | nice #{@program_path}code/parser"
+        collins_params = " #{@program_path}models/model3/grammar 10000 1 1 1 1"
+
+        Dir[in_dir+ "*" + @insuffix].each { |inputfilename|
+
+          STDERR.puts "*** Parsing #{inputfilename} with Collins"
+
+          corpusfilename = File.basename(inputfilename, @insuffix)
+          parsefilename = out_dir + corpusfilename + @outsuffix
+          tempfile = Tempfile.new(corpusfilename)
+
+          # we need to have part of speech tags (but no lemmas at this point)
+          # included automatically by FNTabFormatFile initialize from *.pos
+          tabfile = FNTabFormatFile.new(inputfilename,@pos_suffix)
+
+          CollinsInterface.produce_collins_input(tabfile,tempfile)
+          tempfile.close
+          print collins_prog+" "+tempfile.path+" "+ collins_params+" > "+parsefilename
+          Kernel.system(collins_prog+" "+tempfile.path+" "+
+                        collins_params+" > "+parsefilename)
+          tempfile.close(true)
+        }
+      end
+
+      ###
+      # for a given parsed file:
+      # yield each sentence as a pair
+      #  [SalsaTigerSentence object, FNTabFormatSentence object]
+      # of the sentence in SalsaTigerXML and the matching tab format sentence
+      #
+      # If a parse has failed, returns
+      #  [failed_sentence (flat SalsaTigerSentence), FNTabFormatSentence]
+      # to allow more detailed accounting for failed parses
+      def each_sentence(parsefilename)
+
+        # sanity checks
+        unless @tab_dir
+          raise "Need to set tab directory on initialization"
+        end
+
+        # get matching tab file for this parser output file
+        parserfile = File.new(parsefilename)
+        tabfilename = @tab_dir+File.basename(parsefilename, @outsuffix)+ @insuffix
+
+        corpusfile = FNTabFormatFile.new(tabfilename, @pos_suffix, @lemma_suffix)
+
+        corpusfile.each_sentence {|tab_sent| # iterate over corpus sentences
+
+          my_sent_id = tab_sent.get_sent_id
+
+          while true # find next matching line in parse file
+            line = parserfile.gets
+            # search for the next "relevant" file or end of the file
+            if line.nil? or line=~/^\(TOP/
+              break
+            end
+          end
+          STDERR.puts line
+          # while we search a parse, the parse file is over...
+          if line.nil?
+            raise "Error: premature end of parser file!"
+          end
+
+          line.chomp!
+
+          # it now holds that line =~ ^(TOP
+
+          case line
+          when /^\(TOP~/ # successful parse
+
+            st_sent = STXML::SalsaTigerSentence.empty_sentence(my_sent_id.to_s)
+
+            build_salsatiger(line,st_sent)
+
+            yield [st_sent, tab_sent, CollinsInterface.standard_mapping(st_sent, tab_sent)]
+
+          else
+            # failed parse: create a "failed" parse object
+            # with one nonterminal node and all the terminals
+
+            sent = CollinsInterface.failed_sentence(tab_sent,my_sent_id)
+            yield [sent, tab_sent, CollinsInterface.standard_mapping(sent, tab_sent)]
+
+          end
+        }
+        # after the end of the corpusfile, check if there are any parses left
+        while true
+          line = parserfile.gets
+          if line.nil? # if there are none, everything is fine
+            break
+          elsif line =~ /^\(TOP/ # if there are, raise an exception
+            raise "Error: premature end of corpus file!"
+          end
+        end
+      end
+
+      ###
+      # write Salsa/TIGER XML output to file
+      def to_stxml_file(infilename,  # string: name of parse file
+                        outfilename) # string: name of output stxml file
+
+        outfile = File.new(outfilename, "w")
+        outfile.puts STXML::SalsaTigerXMLHelper.get_header
+        each_sentence(infilename) { |st_sent, tabsent|
+          outfile.puts st_sent.get
+        }
+        outfile.puts STXML::SalsaTigerXMLHelper.get_footer
+        outfile.close
+      end
+
+
+      ########################
+      private
+
+      # Build a SalsaTigerSentence corresponding to the Collins parse in argument string.
+      #
+      # Special features: removes unary nodes and traces
+      def build_salsatiger(string,st_sent)
+
+        nt_c = Counter.new(500)
+        t_c = Counter.new(0)
+
+        position = 0
+        stack = []
+
+        while position < string.length
+          if string[position,1] == "(" # push nonterminal
+            nextspace = string.index(" ",position)
+            nonterminal = string[position+1..nextspace-1]
+            stack.push nonterminal
+            position = nextspace+1
+          elsif string[position,1] == ")" # reduce stack
+            tempstack = []
+            while true
+              # get all Nodes from the stack and put them on a tempstack,
+              # until you find a String, which is a not-yet existing nonterminal
+              object = stack.pop
+              if object.is_a? ::STXML::SynNode
+                tempstack.push(object) # terminal or subtree
+              else #  string (nonterminal label)
+                if tempstack.length == 1 # skip unary nodes: do nothing and write tempstack back to stack
+                  stack += tempstack
+                  break
+                  # puts "Unary node #{object}"
+                end
+                nt_a = object.split("~")
+                unless nt_a.length == 4
+                  # something went wrong. maybe it's about character encoding
+                  if nt_a.length > 4
+                    # yes, assume it's about character encoding
+                    nt_a = [nt_a[0], nt_a[1..-3].join("~"), nt_a[-2], nt_a[-1]]
+                  else
+                    # whoa, _less_ pieces than expected: problem.
+                    $stderr.puts "Collins parse tree translation nonrecoverable error:"
+                    $stderr.puts "Unexpectedly too few components in nonterminal " + nt_a.join("~")
+                    raise StandardError.new("nonrecoverable error")
+                  end
+                end
+
+                # construct a new nonterminal
+                node = st_sent.add_syn("nt",
+                                       STXML::SalsaTigerXMLHelper.escape(nt_a[0].strip), # cat
+                                       nil, # word (doesn't matter)
+                                       nil, # pos (doesn't matter)
+                                       nt_c.next.to_s)
+                node.set_attribute("head",STXML::SalsaTigerXMLHelper.escape(nt_a[1].strip))
+                tempstack.reverse.each {|child|
+                  node.add_child(child,nil)
+                  child.set_parent(node,nil)
+                }
+                stack.push(node)
+                break # while
+              end
+            end
+            position = position + 2 # == nextspace+1
+          else # terminal
+            nextspace = string.index(" ",position)
+            terminal = string[position..nextspace].strip
+            t_a = terminal.split("/")
+            unless t_a.length == 2
+              raise "[collins] Cannot split terminal #{terminal} into word and POS!"
+            end
+
+            word = t_a[0]
+            pos = t_a[1]
+
+            unless pos =~ /TRACE/
+              # construct a new terminal
+              node = st_sent.add_syn("t",
+                                     nil,
+                                     STXML::SalsaTigerXMLHelper.escape(CollinsInterface.unescape(word)), # word
+                                     STXML::SalsaTigerXMLHelper.escape(pos), # pos
+                                     t_c.next.to_s)
+              stack.push(node)
+            end
+            position = nextspace+1
+          end
+        end
+
+        # at the very end, we need to have exactly one syntactic root
+
+        if stack.length != 1
+          raise "[collins] Error: Sentence has #{stack.length} roots"
+        end
+      end
+
+
+      ####
+      # extract the Collins parser input format from a TabFormat object
+      # that includes part-of-speech (pos)
+      #
+      def CollinsInterface.produce_collins_input(corpusfile,tempfile)
+        corpusfile.each_sentence {|s|
+          words = []
+          s.each_line_parsed {|line_obj|
+            word = line_obj.get("word")
+            tag = line_obj.get("pos")
+            if tag.nil?
+              raise "Error: FNTabFormat object not tagged!"
+            end
+            word_tag_pair = CollinsInterface.escape(word,tag)
+            if word_tag_pair =~ /\)/
+              puts word_tag_pair
+              puts s.to_s
+            end
+            words << word_tag_pair
+          }
+          tempfile.puts words.length.to_s+" "+words.join(" ")
+        }
+      end
+
+      ####
+      def CollinsInterface.escape(word,pos) # returns array word+" "+lemma
+        case word
+
+        # replace opening or closing brackets
+        # word representation is {L,R}R{B,S,C} (bracket, square, curly)
+        # POS for opening brackets is LRB, closing brackets RRB
+
+        when "("
+          return "LRB -LRB-"
+        when "["
+          return "LRS -LRB-"
+        when "{"
+          return "LRC -LRB-"
+
+        when ")"
+          return "RRB -RRB-"
+        when "]"
+          return "RRS -RRB-"
+        when "}"
+          return "RRC -RRB-"
+
+        # catch those brackets or slashes inside words
+        else
+          word.gsub!(/\(/,"LRB")
+          word.gsub!(/\)/,"RRB")
+          word.gsub!(/\[/,"LRS")
+          word.gsub!(/\]/,"RRS")
+          word.gsub!(/\{/,"LRC")
+          word.gsub!(/\}/,"RRC")
+          word.gsub!(/\//,"&Slash;")
+
+          word + " " + pos
+        end
+      end
+
+      ####
+      # replace replacements with original values
+      def CollinsInterface.unescape(word)
+        word.gsub(/LRB/,"(").gsub(/RRB/,")").gsub(/LRS/,"[").gsub(/RRS/,"]").gsub(/LRC/,"{").gsub(/RRC/,"}").gsub(/&Slash;/,"/")
+      end
+    end
+  end
+end

data/lib/frappe/interfaces/counter.rb

@@ -0,0 +1,19 @@
+# Counter class - provides unique ids with state
+module Shalmaneser
+  module Frappe
+    class Counter
+      def initialize(init_value)
+        @v = init_value
+      end
+
+      def get
+        @v
+      end
+
+      def next
+        @v += 1
+        @v - 1
+      end
+    end
+  end
+end

data/lib/frappe/interfaces/stanford_interface.rb

@@ -0,0 +1,353 @@
+#-*- coding: utf-8 -*-
+# @author Andrei Beliankou
+# @date> 2013-12-26
+
+####
+# sp 21 07 05
+#
+# modified ke 30 10 05: adapted to fit into SynInterface
+#
+# represents a file containing Stanford parses
+#
+# underlying data structure for individual sentences: SalsaTigerSentence
+
+require_relative 'counter'
+
+require 'salsa_tiger_xml/salsa_tiger_sentence'
+require 'salsa_tiger_xml/salsa_tiger_xml_helper'
+require 'tabular_format/fn_tab_format_file'
+
+require "tempfile"
+################################################
+# Interface class
+module Shalmaneser
+  module Frappe
+    class StanfordInterface < SynInterfaceSTXML
+      LOGGER.debug 'Announcing Stanford Interface'
+      StanfordInterface.announce_me
+
+      def self.system
+        'stanford'
+      end
+
+      def self.service
+        'parser'
+      end
+
+      ###
+      # initialize to set values for all subsequent processing
+      # @param program_path [String] path to a system
+      # @param insuffix [String] suffix of tab files
+      # @param outsuffix [String] suffix of parsed files
+      # @param stsuffix [String] suffix of Salsa/TigerXML files
+      # @param var_hash [Hash] optional arguments
+      def initialize(program_path, insuffix, outsuffix, stsuffix, var_hash = {})
+        super
+
+        # @todo This should be checked in the OptionParser.
+        unless @program_path =~ /\/$/
+          @program_path += '/'
+        end
+
+        # new: evaluate var hash
+        @pos_suffix = var_hash["pos_suffix"]
+        @lemma_suffix = var_hash["lemma_suffix"]
+        @tab_dir = var_hash["tab_dir"]
+
+        # sanity checks
+        # AB: @todo Move this check to the invoker!
+        unless @tab_dir
+          raise "Need to set tab directory on initialization"
+        end
+      end
+
+      ####
+      # parse a directory with TabFormat files and write the parse trees to outputdir
+      # I assume that the files in inputdir are smaller than
+      # the maximum number of sentences that
+      # Stanford can parse in one go (i.e. that they are split)
+      #
+      # @param in_dir [String] input directory name
+      # @param out_dir [String] output directory name
+      def process_dir(in_dir, out_dir)
+
+        # We use the old paradigm for now: the parser binary is wrapped
+        # into a shell script, we invoke this script.
+        #stanford_prog = "#{@program_path}lexparser-german.sh"
+
+        # Borrowed from <lexparser-german.sh>.
+        tlp = 'edu.stanford.nlp.parser.lexparser.NegraPennTreebankParserParams'
+
+        lang_opts = '-hMarkov 1 -vMarkov 2 -vSelSplitCutOff 300 -uwm 1 -unknownSuffixSize 2 -nodeCleanup 2'
+
+        # grammar1 = 'edu/stanford/nlp/models/lexparser/germanPCFG.ser.gz'
+        grammar2 = 'edu/stanford/nlp/models/lexparser/germanFactored.ser.gz'
+
+        stanford_prog = %Q{
+    java -cp "#{@program_path}*:" edu.stanford.nlp.parser.lexparser.LexicalizedParser -maxLength 100 \
+    -tLPP #{tlp} #{lang_opts} -tokenized \
+    -encoding UTF-8 \
+    -outputFormat "oneline" \
+    -outputFormatOptions "includePunctuationDependencies" \
+    -loadFromSerializedFile #{grammar2} \
+    }
+
+        Dir[in_dir + "*" + @insuffix].each do |inputfilename|
+
+          STDERR.puts "*** Parsing #{inputfilename} with StanfordParser."
+          corpusfilename = File.basename(inputfilename, @insuffix)
+          parsefilename = out_dir + corpusfilename + @outsuffix
+          tempfile = Tempfile.new(corpusfilename)
+
+          # we need neither lemmata nor POS tags; stanford can do with the words
+          corpusfile = FNTabFormatFile.new(inputfilename, nil, nil)
+
+          corpusfile.each_sentence do |sentence|
+            #puts sentence
+            tempfile.puts sentence
+          end
+
+          tempfile.close
+
+          # Invoke the expternal parser.
+          invocation_str = "#{stanford_prog} #{tempfile.path} > #{parsefilename} 2>/dev/null"
+          STDERR.puts invocation_str
+
+          Kernel.system(invocation_str)
+
+        end
+      end
+
+      ###
+      # for a given parsed file:
+      # yield each sentence as a pair
+      #  [SalsaTigerSentence object, FNTabFormatSentence object]
+      # of the sentence in SalsaTigerXML and the matching tab format sentence
+      #
+      # If a parse has failed, returns
+      #  [failed_sentence (flat SalsaTigerSentence), FNTabFormatSentence]
+      # to allow more detailed accounting for failed parses
+      # (basically just a flat structure with a failed=true attribute
+      # at the sentence node)
+      def each_sentence(parsefilename)
+
+        # get matching tab file for this parser output file
+        parsefile = File.new(parsefilename)
+        tabfilename = @tab_dir + File.basename(parsefilename, @outsuffix) + @insuffix
+        tabfile = FNTabFormatFile.new(tabfilename, @postag_suffix, @lemma_suffix)
+
+        sentid = 0
+        tabfile.each_sentence do |tab_sent| # iterate over corpus sentences
+
+          # assemble next sentence in Stanford file by reading lines from parsefile
+          # for stanford:
+          while true
+            sentence_str = parsefile.gets
+            # Sentence contains a valid or an empty parse.
+            # AB: @todo Investigate how does an empty parse look like.
+            if sentence_str =~ /\(ROOT|TOP|PSEUDO/ or sentence_str =~ /^\(\(\)/
+              sentid +=1
+              break
+            # There is no parse.
+            elsif sentence_str.nil?
+              raise "Error: premature end of parser file!"
+            end
+          end
+
+          sentence_str.chomp!.gsub!(/\)\)/, ') )').gsub!(/\)\)/, ') )')
+
+          # VAFIN_HD -> VAFIN-HD
+          # for the current german grammar not really usefull
+          #sentence_str.gsub!(/(\([A-Z]+)_/, '\1-')
+
+          if tab_sent.get_sent_id == "--"
+            my_sent_id = "#{File.basename(parsefilename, @outsuffix)}_#{sentid}"
+          else
+            my_sent_id = tab_sent.get_sent_id
+          end
+
+          st_sent = build_salsatiger(" " + sentence_str + " ", 0,
+                                     [], Counter.new(0),
+                                     Counter.new(500),
+                                     SalsaTigerSentence.empty_sentence(my_sent_id.to_s))
+
+          # AB: When is it possible?
+          next unless st_sent
+
+          yield [st_sent, tab_sent, StanfordInterface.standard_mapping(st_sent, tab_sent)]
+        end
+
+        # All TabFile sentences are consumed.
+        # Now we may just encounter comments, garbage, empty lines etc.
+        while abline = parsefile.gets
+          case abline
+          when /^%/, /^\s*$/
+          # Found empty lines, comments, end of input indicate end of
+          # current parse.
+          # AB: TODO Investigate what can StanfordParser output.
+          else
+            # We found something meaningfull, a parse tree.
+            raise "Error: Premature end of tab file! Found line: #{abline}"
+          end
+        end
+
+        parsefile.close
+      end # each_sentence
+
+
+      ###
+      # write Salsa/TIGER XML output to file
+      # @param infilename [String] name of parse file
+      # @param outfilename [String] name of output stxml file
+      def to_stxml_file(infilename, outfilename)
+
+        File.open(outfilename, 'w') do |outfile|
+          outfile.puts STXML::SalsaTigerXMLHelper.get_header
+          each_sentence(infilename) do |st_sent, tabsent|
+            outfile.puts st_sent.get
+          end
+          outfile.puts STXML::SalsaTigerXMLHelper.get_footer
+        end
+
+      end
+
+
+
+      ########################
+      private
+
+      ###
+      # Recursive function for parsing a Stanford parse tree and
+      # building a SalsaTigerSentence recursively
+      #
+      # Algorithm: manage stack which contains, for the current constituent,
+      # child constituents (if a nonterminal), and the category label.
+      # When the end of a constituent is reached, a new SynNode (TigerSalsa node) ist created.
+      # All children and the category label are popped from the stack and integrated into the
+      # TigerSalsa data structure. The new node is re-pushed onto the
+      # stack.
+      # @param sentence [String]
+      # @param pos [Fixnum] position in string (index)
+      # @param stack [Array] stack with incomplete nodes
+      # @param termc [Counter] terminal counter
+      # @param nontc [Counter] nonterminal counter
+      # @param sent_obj [SalsaTigerSentence] SalsaTigerSentence
+      def build_salsatiger(sentence, pos, stack, termc, nontc, sent_obj)
+
+        if sentence =~ /\(\)/
+          return nil
+        end
+
+        # main case distinction: match the beginning of our string
+        # (i.e. what follows our current position in the string)
+        case sentence[pos..-1]
+        when /^ *$/ # nothing -> whole sentence parsed
+          if stack.length == 1
+            # sleepy always delivers one "top" node; if we don't get just one
+            # node, something has gone wrong
+            node = stack.pop
+            node.del_attribute("gf")
+            return sent_obj
+          else
+            raise "Error: more than one root node (stack length #{stack.length}). Full sentence: \n#{sentence}"
+          end
+
+        when /^\s*\(([^ )]+) /
+          # match the beginning of a new constituent
+          # (opening bracket + category + space, may not contain closing bracket)
+          cat = $1
+          if cat.nil? or cat == ""
+            raise "Error: found category nil in sentence #{sentence[pos,10]}, full sentence\n#{sentence}"
+          end
+          #          STDERR.puts "new const #{cat}"
+          stack.push cat # throw the category label on the stack
+          return build_salsatiger(sentence, pos + $&.length, stack, termc, nontc, sent_obj)
+
+        when /^\s*(\S+)\) /
+          # match the end of a terminal constituent (something before a closing bracket + space)
+          word = $1
+
+          comb_cat = stack.pop
+          if comb_cat.to_s == ""
+            raise "Empty cat at position #{sentence[pos,10]}, full sentence\n#{sentence}"
+          end
+
+          cat, gf = split_cat(comb_cat)
+          node = sent_obj.add_syn("t",
+                                  nil,  # cat (doesn't matter here)
+                                  STXML::SalsaTigerXMLHelper.escape(word), # word
+                                  cat,  # pos
+                                  termc.next.to_s)
+          node.set_attribute("gf", gf)
+          #          STDERR.puts "completed terminal #{cat}, #{word}"
+          stack.push node
+          return build_salsatiger(sentence, pos + $&.length, stack, termc, nontc, sent_obj)
+
+        when /^\s*\)/ # match the end of a nonterminal (nothing before a closing bracket)
+          # now collect children:
+          # pop items from the stack until you find the category
+          children = []
+          while true
+            if stack.empty?
+              raise "Error: stack empty; cannot find more children"
+            end
+
+            item = stack.pop
+            # @todo Change the check from string to class instances. 'SynNode' -> SynNode
+            case item
+            when STXML::SynNode # this is a child
+              children.push item
+            when String # this is the category label
+              if item.to_s == ""
+                raise "Empty cat at position #{sentence[pos,10]}, full sentence\n#{sentence}"
+              end
+              cat, gf = split_cat(item)
+              break
+            else
+              raise "Error: unknown item class #{item.class}."
+            end
+          end
+
+          # now add a nonterminal node to the sentence object and
+          # register the children nodes
+          node = sent_obj.add_syn("nt",
+                                  cat, # cat
+                                  nil, # word (doesn't matter)
+                                  nil, # pos (doesn't matter)
+                                  nontc.next.to_s)
+
+          children.each do |child|
+            child_gf = child.get_attribute("gf")
+            child.del_attribute("gf")
+            node.add_child(child,child_gf)
+            child.add_parent(node, child_gf)
+          end
+
+          node.set_attribute("gf", gf)
+          #          STDERR.puts "Completed nonterm #{cat}, #{children.length} children."
+          stack.push node
+
+          return build_salsatiger(sentence, pos + $&.length, stack,termc, nontc, sent_obj)
+        else
+          raise "Error: cannot analyse sentence at pos #{pos}: #{sentence[pos..-1]}. Complete sentence: \n#{sentence}"
+        end
+      end
+
+      ###
+      # StanfordParser delivers node labels as "phrase type"-"grammatical function",
+      # but the GF may not be present.
+      # @param cat [String]
+      # @return [Array]
+      def split_cat(cat)
+
+        md = cat.match(/^([^-]*)(-([^-]*))?$/)
+        raise "Error: Could not identify category in #{cat}!" unless md[1]
+
+        proper_cat = md[1]
+        gf = md[3] ? md[3] : ''
+
+        [proper_cat, gf]
+      end
+    end
+  end
+end