frprep 0.0.1.prealpha

data/lib/frprep/Parser.rb

@@ -0,0 +1,213 @@
+# Alexander Koller 2003
+# extended Katrin Erk June 2003
+#
+# Classes that return a list of sentence DOMs, from various sources
+#
+# Each class in this file defines the following methods:
+#
+#   initialize(...)     "..." depends on the class
+#   extractDOMs()       return list of all s nodes as DOM objects
+#   each_s()            iterate over s nodes; may take less memory  
+
+
+require "rexml/document"
+
+class FileParser 
+
+  include REXML
+
+  def initialize(filename)
+    @file = File.new(filename)
+    @doc = nil
+  end
+
+  # returns an array of DOMs for the sentences
+  def extractDOMs()
+    ensureParsedDocument()
+    @doc.get_elements("/corpus/body/s")
+  end
+
+  # Iterates over all sentence nodes. This may be more memory
+  # efficient than using extractDOMs(), but isn't in this case.
+  def each_s()
+    extractDOMs().each { |dom| yield(dom) }
+  end
+
+  # Iterates over all sentence nodes. The block passed to this
+  # method should return a DOM object as a value. After the iteration
+  # has been completed, the contents of /corpus/body are then replaced
+  # by the list of these results.
+  # At the moment, this changes the FileParser object. This should
+  # probably change in the future, but I don't want to mess with
+  # cloning now.
+  def process_s!()
+    newBody = Element.new('body')
+    each_s { |dom| newBody.add_element( yield(dom) ) }
+
+    @doc.delete_element("/corpus/body")
+    @doc.elements["corpus"].add_element(newBody)
+
+    return @doc
+  end
+
+
+
+  private
+  
+  def ensureParsedDocument()
+    if @doc == nil then
+      @doc = Document.new(@file)
+    end
+  end
+
+      
+end
+
+
+
+
+#####################################################################
+
+
+
+
+class FilePartsParser
+  # @file = File object for the corpus
+  # @head = string up to the first <s> tag
+  # @tail = string after the last </s> tag
+  # @rest = string starting with the latest <s> tag (complete this to
+  # a <s>...</s> structure by reading up to next </s> tag)
+  # @readCompletely = boolean specifying whether there's still something
+  # left to read in the file
+
+  attr_reader :head, :tail
+
+  def initialize(filename)
+    @file = File.new(filename)
+    @readCompletely = false
+    # read stuff into @head and initialize @rest
+    @head = ''
+    begin
+      while true do
+	line = @file.readline() 
+	if line =~ /(.*)(<s\s.*)/ then
+	  @head = @head << $1
+	  @rest = $2
+	  break
+	elsif line =~ /^(.*)(<\/body[\s>].*)$/
+	  # empty corpus
+	  @head = @head << $1
+	  @tail = $2
+	  while (line = @file.readline())
+	    @tail << "\n" + line
+	  end
+	  @readCompletely = true
+	  break
+	else
+	  @head = @head << line
+	end
+      end
+    rescue EOFError
+      @readCompletely = true
+    end
+  end
+
+  def close()
+    @file.close()
+  end
+
+  def extractDOMs()
+    allDOMs = Array.new
+
+    process_s!() { |dom| 
+      allDOMs.push(dom) 
+      Element.new("x")
+    }
+    return allDOMs
+  end
+
+  def each_s()
+    process_s!() { |dom| 
+      yield(dom) 
+      Element.new("x")
+    }
+  end
+
+  # This function returns the string for the modified corpus.
+  # It doesn't change the internal state of the FilePartsParser,
+  # and is much more memory (and probably time) efficient than
+  # FileParser#process_s!.
+  # The block that is called by the method is given an element
+  # as its argument and is expected to return a changed element.
+  def process_s!()
+    if @readCompletely
+      return
+    end
+
+    ret = ''
+    scan_s() { |element|
+      # Process the <s> ... </s> element
+      doc = Document.new(element)
+      elt = doc.root
+      changedElt = yield(elt)
+
+      changedEltAsString = ''
+      changedElt.write(changedEltAsString, 0)
+      ret <<= changedEltAsString
+    }
+
+    return ret
+  end
+
+  # KE 12.6.03: scan_s : 
+  # doesn't parse a sentence before yielding it
+  # doesn't allow for any changes
+  # but otherwise the same as process_s!
+  def scan_s()
+    if @readCompletely
+      return
+    end
+     
+    begin
+      while true do
+	# Invariant: At this point, @rest always starts with an
+	# unseen <s> tag.
+	
+	# First, we continue reading until we find the closing </s>
+	# No exception should occur in this loop if we're parsing
+	# a valid XML document.
+	while @rest !~ /^(.*<\/s>)(.*)/m do
+	  @rest = @rest << @file.readline()
+	end
+
+	element = $1
+	@rest = $2
+
+	yield(element) # change HERE: element not parsed!
+
+	# Read on up to the next <s>
+	while @rest !~ /(.*)(<s\s.*)/m do
+	  @rest = @rest << @file.readline()
+	end
+
+	@rest = $2
+      end
+    rescue EOFError
+      @tail = @rest
+      @readCompletely = true
+    end
+  end
+
+  # KE 5.11.03: get_rest: read all of the file not processed up to this point
+  # and return it as a string
+  def get_rest()
+    begin
+      while true do
+	@rest = @rest << @file.readline()
+      end
+    rescue EOFError
+      @readCompletely = true
+    end
+    return @rest
+  end
+end

data/lib/frprep/RegXML.rb

@@ -0,0 +1,269 @@
+# RegXML
+#
+# Katrin Erk June 2005
+
+# SalsaTigerRegXML: take control of the data structure, no underlying xml
+# representation anymore, re-generation of xml on demand
+
+class RegXML
+
+  def initialize(string, # string representing a single XML element
+                 i_am_text = false) # boolean: xml element (false) or text (true)
+
+    unless string.class == String
+      raise "First argument to RegXML.new must be string. I got #{string.class.to_s}"
+    end
+    if i_am_text
+      @s = string
+      @i_am_text = true
+    else
+      @s = string.gsub(/\n/,  " ").freeze
+      @i_am_text = false
+
+      element_test()
+      dyck_test()
+    end
+  end
+
+  def to_s()
+    return xml_readable(@s)
+  end
+  
+  def text?
+    return @i_am_text
+  end
+
+  def name()
+    if @i_am_text
+      # text
+      return nil
+
+    else
+      # xml element
+      if @s =~ /^\s*<\s*([\w-]+)[\s\/>]/
+        return $1
+      else
+        raise "Cannot parse:\n#{xml_readable(@s)}"
+      end
+    end
+  end
+
+  def attributes()
+    if @i_am_text
+      # text
+      return {}
+
+    else
+      #  xml element
+
+      # remove <element_name  from the beginning of @s,
+      # place the rest up to the first > into elt_contents:
+      # this is a string of the form
+      # - either (name=value)*
+      # - or     (name=value)*/
+      unless @s =~ /^\s*<\s*#{name()}(.*)$/
+        raise "Cannot parse:\n #{xml_readable(@s)}"
+      end
+      
+      retv = Hash.new
+      elt_contents = $1
+      
+      # repeat until only > or /> is left
+      while elt_contents !~ /^\s*\/?>/
+        
+        # shave off the next name=value pair
+        # put the rest into elt_contents
+        # make sure that if the value is quoted with ',
+        # we accept " inside the value, and vice versa.
+        unless elt_contents =~ /^\s*([\w-]+)=(['"])(.*?)\2(.*)$/
+          raise "Cannot parse:\n #{xml_readable(elt_contents)}"
+        end
+        retv[$1] = $3
+        elt_contents = $4
+      end
+
+      return retv
+    end
+  end
+
+  def children_and_text()
+    if @i_am_text
+      return []
+
+    else
+      if unary_element()
+        # <bla/>, no children
+        return []
+      end
+      
+      # @s has the form <bla...>  ... </bla>.
+      # remove <bla ...>  from the beginning of @s,
+      # place the rest up to </bla> into children_s:
+      
+      mainname = name()
+      unless @s =~ /^\s*<\s*#{mainname}(\s+[\w-]+=(["']).*?\2)*\s*>(.*?)<\/\s*#{mainname}\s*>\s*$/
+        raise "Cannot parse:\n #{xml_readable(@s)}"
+      end
+      
+      retv = Array.new
+      children_s = $3
+
+      # repeat until only whitespace is left
+      while children_s !~ /^\s*$/
+        
+      # shave off the next bit of text
+        # put the rest into children_s
+        unless children_s =~ /^\s*(.*?)(<.*$|$)/
+          $stderr.puts "Whole was:\n #{xml_readable(@s)}"
+          $stderr.puts 
+          raise "Cannot parse:\n #{xml_readable(children_s)}"
+        end
+        unless $1.strip.empty?
+          children_s = $2
+          retv << RegXML.new($1, true)
+        end
+        
+        # anything left after we've parsed text?
+        if children_s =~ /^s*$/
+          break
+        end
+
+        # shave off the next child
+        # and put the rest into children_s
+
+        # determine the next child's name, and the string index at which
+        # the element start tag ends with either / or >
+        unless children_s =~ /^\s*(<\s*([\w-]+)(\s+[\w-]+=(["']).*?\4)*\s*)/
+          $stderr.puts "Whole was:\n #{xml_readable(@s)}"
+          $stderr.puts 
+          raise "Cannot parse:\n #{xml_readable(children_s)}"
+        end
+        childname = $2
+        child = $1
+        endofelt_ix = $&.length()
+
+        
+        # and remove it
+        case children_s[endofelt_ix..-1]
+        when /^\/>(.*)$/
+          # next child is a unary element
+          children_s = $1
+          retv << RegXML.new(child + "/>")
+
+        when /^(>.*?<\s*\/\s*#{childname}\s*>)(.*)$/
+          children_s = $2
+          retv << RegXML.new(child + $1)
+
+        else
+          $stderr.puts "Whole was:\n #{xml_readable(@s)}"
+          $stderr.puts 
+          raise "Cannot parse:\n#{xml_readable(children_s)}"
+        end
+      end
+
+      return retv
+    end
+  end
+
+  def RegXML.test()
+    bla = RegXML.new("  <bla blupp='a\"b' 
+lalala=\"c\">
+  <lalala> </lalala>
+  texttext
+  <lala blupp='b'/>
+  nochtext
+  <la> <l/> </la>  
+</ bla >
+")
+    puts "name " + bla.name()
+    puts
+    puts bla.to_s()
+    puts
+    bla.attributes.each { |attr, val|
+      puts "attr " + attr + "=" + val
+    }
+    puts
+    bla.children_and_text.each { |child_obj|
+      if child_obj.text?
+        puts "da text " + child_obj.to_s
+      else
+        puts "da child " + child_obj.to_s
+      end
+    }
+    puts
+
+    puts "NEU"
+    bla = RegXML.new("  < bla blupp='a\"'/> ")
+    puts "name " + bla.name()
+    puts
+    puts bla.to_s()
+    puts
+    bla.attributes.each { |attr, val|
+      puts "attr " + attr + "=" + val
+    }
+    puts
+    bla.children_and_text.each { |child_obj|
+      if child_obj.text?
+        puts "da text " + child_obj.to_s
+      else
+        puts "da child " + child_obj.to_s
+      end
+    }
+    puts
+
+  end
+
+  ##############
+  protected
+
+  def unary_element()
+    # <bla/>
+    if @s =~ /^\s*<.*\/>\s*$/
+      return true
+    else
+      return false
+    end
+  end
+
+  def element_test()
+    # make sure we have a single XML element, either <bla/> or
+    # <bla>...</bla>
+    
+    if unary_element()
+      # <bla/>
+    elsif @s =~ /^\s*<\s*([\w-]+)\W.*?<\/\s*\1\s*>\s*$/
+      # <bla  > ... </bla>
+    else
+      raise "Cannot parse:\n #{xml_readable(@s)}"
+    end
+  end
+
+  def dyck_test()
+    # every prefix of @s must have at least as many < as >
+    opening = 0
+    closing = 0
+    @s.scan(/[<>]/) { |bracket|
+      case bracket
+      when "<"
+        opening += 1
+      when ">"
+        closing += 1
+        if closing > opening
+          raise "More closing than opening brackets in prefix of:\n #{xml_readable(@s)}"
+        end
+      end
+    }
+
+    # and in total, @s must have equally many < and >
+    unless @s.count("<") == @s.count(">")
+      raise "Inequal number of brackets in:\n #{xml_readable(@s)}"
+    end
+  end
+
+  def xml_readable(string)
+    return string.gsub(/>/, ">\n")
+  end
+end
+
+# RegXML.test()
+