shalmaneser-lib 1.2.rc5

data/lib/monkey_patching/array.rb

@@ -0,0 +1,82 @@
+require_relative 'enumerable_bool'
+require_relative 'enumerable_distribute'
+require_relative 'subsumed'
+
+# Extensions for the class Array.
+class Array
+  include EnumerableBool
+  include EnumerableDistribute
+  include Subsumed
+
+  ###
+  # interleave N arrays:
+  # given arrays [a1... an], [b1,...,bn], ..[z1, ...,zn]
+  # return [[a1,b1, .., z1]...,[an,bn, .., zn]]
+  #
+  # if one array is longer than the other,
+  # e.g. [a1...an], [b1,...,bm] with n> m
+  # the result is
+  # [[a1,b1],...[am, bm], [am+1, nil], ..., [an, nil]]
+  # and analogously for m>n
+  def interleave(*arrays)
+    len = [length, arrays.map(&:length).max].max
+    (0..len-1).to_a.map do |ix|
+      [at(ix)] + arrays.map { |a| a[ix] }
+    end
+  end
+
+  ###
+  # count the number of occurrences of element in this array
+  def count(element)
+    num = 0
+    each { |my_element|
+      if my_element == element
+        num += 1
+      end
+    }
+
+    num
+  end
+
+  ###
+  # count the number of occurrences of
+  # elements from list in this array
+  def counts(list)
+    num = 0
+    each { |my_element|
+      if list.include? my_element
+        num += 1
+      end
+    }
+    return num
+  end
+
+  ###
+  # draw a random sample of size N
+  # from this array
+  def sample(size)
+    if size < 0
+      return nil
+    elsif size == 0
+      return []
+    elsif size >= length
+      return self.clone
+    end
+
+    rank = {}
+    each { |my_element|
+      rank[my_element] = rand
+    }
+    return self.sort { |a, b| rank[a] <=> rank[b] }[0..size-1]
+  end
+
+  def map_with_index(&block)
+    retv = []
+
+    each_with_index { |x, index|
+      retv << block.call(x, index)
+    }
+
+    return retv
+  end
+end

data/lib/monkey_patching/enumerable_bool.rb

@@ -0,0 +1,24 @@
+################
+module EnumerableBool
+  ###
+  # And_(x \in X) block(x)
+  def big_and(&block)
+    each do |x|
+      unless block.call(x)
+        return false
+      end
+    end
+
+    true
+  end
+
+  ###
+  # Sum_(x \in X) block(x)
+  def big_sum(init = 0, &block)
+    sum = init
+    block = proc { |x| x } unless block_given?
+    each { |x| sum += block.call(x) }
+
+    sum
+  end
+end

data/lib/monkey_patching/enumerable_distribute.rb

@@ -0,0 +1,18 @@
+################
+# Given an enumerable, distribute its items into two bins (arrays)
+# depending on whether the block returns true
+module EnumerableDistribute
+  def distribute(&block)
+    retv1 = []
+    retv2 = []
+    each do |x|
+      if block.call(x)
+        retv1 << x
+      else
+        retv2 << x
+      end
+    end
+
+    [retv1, retv2]
+  end
+end

data/lib/monkey_patching/file.rb

@@ -0,0 +1,131 @@
+require 'fileutils'
+
+# Extensions for the class File.
+class File
+  ########
+  # check whether a given path exists,
+  # and if it doesn't, make sure it is created.
+  #
+  # piece together the strings in 'pieces' to make the path,
+  # appending "/" to all strings if necessary
+  #
+  # returns: the path pieced together
+  # strings, to be pieced together
+  def self.new_dir(*pieces)
+    dir_path, _dummy = File.make_path(pieces, true)
+
+    unless File.exist?(dir_path)
+      FileUtils.mkdir_p(dir_path)
+    end
+    # check that all went well in creating the directory)
+    File.existing_dir(dir_path)
+
+    dir_path
+  end
+
+  ########
+  # same as new_dir, but last piece is a filename
+  def self.new_filename(*pieces)
+    dir_path, whole_path = File.make_path(pieces, false)
+
+    unless File.exist?(dir_path)
+      FileUtils.mkdir_p dir_path
+    end
+    # check that all went well in creating the directory)
+    File.existing_dir(dir_path)
+
+    whole_path
+  end
+
+  #####
+  # check whether a given path exists,
+  # and report failure of it does not exist.
+  #
+  # piece together the strings in 'pieces' to make the path,
+  # appending "/" to all strings if necessary
+  #
+  # returns: the path pieced together
+  def self.existing_dir(*pieces) # strings
+    dir_path, _dummy = File.make_path(pieces, true)
+
+    unless File.exist?(dir_path) && File.directory?(dir_path)
+      raise "Error: Directory #{dir_path} doesn't exist."
+    end
+    unless File.executable? dir_path
+      raise "Error: Cannot access directory #{dir_path}."
+    end
+
+    dir_path
+  end
+
+  # @note AB: This method is not used anywhere.
+=begin
+  ####
+  # like existing_dir, but last bit is filename
+  def self.existing_filename(*pieces)
+    dir_path, whole_path = File.make_path(pieces, false)
+
+    unless File.exist?(dir_path) && File.directory?(dir_path)
+      $stderr.puts "Error: Directory #{dir_path} doesn't exist. Exiting"
+      exit(1)
+    end
+
+    unless File.executable?(dir_path)
+      $stderr.puts "Error: Cannot access directory #{dir_path}. Exiting."
+      exit(1)
+    end
+
+    whole_path
+  end
+=end
+
+  ####
+  # piece together the strings in 'pieces' to make a path,
+  # appending "/" to all but the last string if necessary
+  #
+  # if 'pieces' is already a string, take that as a one-piece path
+  #
+  # if dir is true, also append "/" to the last piece of the string
+  #
+  # the resulting path is expanded: For example, initial
+  # ~ is expanded to the setting of $HOME
+  #
+  # returns: pair of strings (directory_part, whole_path)
+  # @param pieces [String, Array]
+  # @param is_dir [True, False, Nil]
+  # @api private
+  def self.make_path(pieces, is_dir = false)
+    if pieces.is_a?(String)
+      pieces = [pieces]
+    end
+
+    dir = ''
+    # iterate over all but the filename
+    if is_dir
+      last_dir_index = -1
+    else
+      last_dir_index = -2
+    end
+    pieces[0..last_dir_index].each { |piece|
+      if piece.nil?
+        # whoops, nil entry in name of path!
+        $stderr.puts "File.make_path ERROR: nil for piece of path name."
+        next
+      end
+      if piece =~ /\/$/
+        dir << piece
+      else
+        dir << piece << "/"
+      end
+    }
+
+    dir = File.expand_path(dir)
+
+    # expand_path removes the final "/" again
+    unless dir =~ %r{/$}
+      dir += "/"
+    end
+
+    is_dir ? [dir, dir] : [dir, dir + pieces[-1]]
+  end
+end

data/lib/monkey_patching/subsumed.rb

@@ -0,0 +1,24 @@
+###
+# extend Array class by subsumption
+module Subsumed
+  # @note This method is used by [RosyConfusability]
+  def subsumed_by?(array2)
+    temp = array2.clone
+
+    self.each { |el|
+      found = false
+      temp.each_index { |ix|
+        if el == temp[ix]
+          temp.delete_at(ix)
+          found = true
+          break
+        end
+      }
+      unless found
+        return false
+      end
+    }
+
+    return true
+  end
+end

data/lib/ruby_class_extensions.rb

@@ -0,0 +1,4 @@
+require_relative 'monkey_patching/file'
+require_relative 'monkey_patching/array'
+require_relative 'monkey_patching/enumerable_bool'
+require_relative 'monkey_patching/enumerable_distribute'

data/lib/salsa_tiger_xml/corpus.rb

@@ -0,0 +1,24 @@
+require 'nokogiri'
+
+module STXML
+  class Corpus
+    attr_reader :doc
+
+    def initialize(filename)
+      @doc = File.open(filename) do |f|
+        Nokogiri::XML(f)
+      end
+    end
+
+    def each_sentence
+      return enum_for(:each_sentence) unless block_given?
+      @doc.xpath('//s').each do |s|
+        yield s
+      end
+    end
+
+    def sentences
+      @doc.xpath('//s')
+    end
+  end
+end

data/lib/salsa_tiger_xml/fe_node.rb

@@ -0,0 +1,98 @@
+require_relative 'sem_node'
+require_relative 'reg_xml'
+
+module STXML
+  #############
+  # class FeNode
+  #
+  # inherits from SemNode,
+  # adds to it methods specific to nodes
+  # that describe a frame element or target
+  #
+  # additional/changed methods:
+  #----------------------------
+  #
+  # name      returns the name of the frame element, or "target"
+  #
+  # add_child, remove_child
+  class FeNode < SemNode
+    ###
+    # either RegXMl object or the name of the FE as a string
+    # string: ID to use if we just got the name of the FE
+    def initialize(name_or_xml, id_if_name = nil)
+      case name_or_xml
+      when String
+        if name_or_xml == "target"
+          super(RegXML.new("<target id=\'#{xml_secure_val(id_if_name.to_s)}\'/>"))
+          @i_am_target = true
+        else
+          super(RegXML.new("<fe name=\'#{xml_secure_val(name_or_xml)}\' id=\'#{xml_secure_val(id_if_name.to_s)}\'/>"))
+          @i_am_target = false
+        end
+      when RegXML
+        super(name_or_xml)
+
+        if name_or_xml.name == "target"
+          @i_am_target = true
+        else
+          @i_am_target = false
+        end
+      else
+        raise "Shouldn't be here: #{name_or_xml.class}."
+      end
+
+      # child_attr: keep additional attributes of <fenode> elements,
+      # if there are any
+      # child_attr: hash syn_node_id(string) -> attributes(hash)
+      @child_attr = {}
+    end
+
+    ###
+    def name
+      if @i_am_target
+        'target'
+      else
+        get_attribute("name")
+      end
+    end
+
+    ###
+    def add_child(syn_node, xml_obj = nil)
+      if xml_obj
+        # we've been given the fenode XML element
+        # see if there are any attributes that we will need:
+        # get attributes, remove the idref (we get that from the
+        # child's ID directly)
+        at = xml_obj.attributes
+        at.delete("idref")
+        unless at.empty?
+          @child_attr[syn_node.id] = at
+        end
+      end
+
+      super(syn_node, nil, "pointer_insteadof_edge" => true)
+    end
+
+    ###
+    def remove_child(syn_node, varhash={})
+      super(syn_node, nil, "pointer_insteadof_edge" => true)
+    end
+
+    protected
+
+    def get_xml_ofchildren
+      return children.map { |child|
+        if @child_attr[child.id]
+          "<fenode idref=\'#{xml_secure_val(child.id)}\'" +
+            @child_attr[child.id].to_a.map { |attr, val|
+            " #{attr}=\'#{xml_secure_val(val)}\'"
+          }.join +
+            "/>\n"
+
+        else
+          "<fenode idref=\'#{xml_secure_val(child.id)}\'/>\n"
+        end
+      }.join
+    end
+  end
+end

data/lib/salsa_tiger_xml/file_parts_parser.rb

@@ -0,0 +1,214 @@
+# Alexander Koller 2003
+# extended Katrin Erk June 2003
+#
+# Classes that return a list of sentence DOMs, from various sources
+#
+# Each class in this file defines the following methods:
+#
+#   initialize(...)     "..." depends on the class
+#   extractDOMs()       return list of all s nodes as DOM objects
+#   each_s()            iterate over s nodes; may take less memory
+
+require "rexml/document"
+
+module STXML
+
+  class FilePartsParser
+    # <@file> = File object for the corpus
+    # <@head> = string up to the first <s> tag
+    # <@tail> = string after the last </s> tag
+    # <@rest> = string starting with the latest <s> tag (complete this to
+    # a <s>...</s> structure by reading up to next </s> tag)
+    # <@readCompletely> = boolean specifying whether there's still something
+    # left to read in the file
+
+    attr_reader :head, :tail
+
+    def initialize(filename)
+      @file = File.new(filename)
+      @readCompletely = false
+      # read stuff into @head and initialize @rest
+      @head = ''
+      begin
+        loop do
+          line = @file.readline
+          if line =~ /(.*)(<s\s.*)/ then
+            @head = @head << $1
+            @rest = $2
+            break
+          elsif line =~ /^(.*)(<\/body[\s>].*)$/
+            # empty corpus
+            @head = @head << $1
+            @tail = $2
+            while (line = @file.readline)
+              @tail << "\n" + line
+            end
+            @readCompletely = true
+            break
+          else
+            # @todo Edit this horror!
+            @head = @head << line
+          end
+        end
+      rescue EOFError
+        @readCompletely = true
+      end
+    end
+
+    def close
+      @file.close
+    end
+
+    # @note AB: This method isn't used anywhere.
+    def extractDOMs
+      allDOMs = []
+
+      process_s! do |dom|
+        allDOMs.push(dom)
+        Element.new("x")
+      end
+
+      allDOMs
+    end
+
+    # @note AB: This method isn't used anywhere.
+    def each_s
+      process_s! do |dom|
+        yield(dom)
+        Element.new("x")
+      end
+    end
+
+    # This function returns the string for the modified corpus.
+    # It doesn't change the internal state of the FilePartsParser,
+    # and is much more memory (and probably time) efficient than
+    # FileParser#process_s!.
+    # The block that is called by the method is given an element
+    # as its argument and is expected to return a changed element.
+    # @note This method isn't used anywhere.
+    def process_s!
+      if @readCompletely
+        return
+      end
+
+      ret = ''
+      scan_s { |element|
+        # Process the <s> ... </s> element
+        doc = Document.new(element)
+        elt = doc.root
+        changedElt = yield(elt)
+
+        changedEltAsString = ''
+        changedElt.write(changedEltAsString, 0)
+        ret <<= changedEltAsString
+      }
+
+      return ret
+    end
+
+    # KE 12.6.03: scan_s :
+    # doesn't parse a sentence before yielding it
+    # doesn't allow for any changes
+    # but otherwise the same as process_s!
+    # @return [String] A String with one xml encoded sentence.
+    def scan_s
+      if @readCompletely
+        return
+      end
+
+      begin
+        while true do
+          # Invariant: At this point, @rest always starts with an
+          # unseen <s> tag.
+
+          # First, we continue reading until we find the closing </s>
+          # No exception should occur in this loop if we're parsing
+          # a valid XML document.
+          while @rest !~ /^(.*<\/s>)(.*)/m do
+            @rest = @rest << @file.readline
+          end
+
+          element = $1
+          @rest = $2
+
+          yield(element) # change HERE: element not parsed!
+
+          # Read on up to the next <s>
+          while @rest !~ /(.*)(<s\s.*)/m do
+            @rest = @rest << @file.readline
+          end
+
+          @rest = $2
+        end
+      rescue EOFError
+        @tail = @rest
+        @readCompletely = true
+      end
+    end
+
+    # KE 5.11.03: get_rest: read all of the file not processed up to this point
+    # and return it as a string
+    def get_rest
+      begin
+        loop do
+          @rest = @rest << @file.readline
+        end
+      rescue EOFError
+        @readCompletely = true
+      end
+      return @rest
+    end
+  end
+end
+
+# This part seems to be obsolete, delete it!
+=begin
+
+class FileParser
+
+  include REXML
+
+  def initialize(filename)
+    @file = File.new(filename)
+    @doc = nil
+  end
+
+  # returns an array of DOMs for the sentences
+  def extractDOMs()
+    ensureParsedDocument()
+    @doc.get_elements("/corpus/body/s")
+  end
+
+  # Iterates over all sentence nodes. This may be more memory
+  # efficient than using extractDOMs(), but isn't in this case.
+  def each_s()
+    extractDOMs().each { |dom| yield(dom) }
+  end
+
+  # Iterates over all sentence nodes. The block passed to this
+  # method should return a DOM object as a value. After the iteration
+  # has been completed, the contents of /corpus/body are then replaced
+  # by the list of these results.
+  # At the moment, this changes the FileParser object. This should
+  # probably change in the future, but I don't want to mess with
+  # cloning now.
+  def process_s!()
+    newBody = Element.new('body')
+    each_s { |dom| newBody.add_element( yield(dom) ) }
+
+    @doc.delete_element("/corpus/body")
+    @doc.elements["corpus"].add_element(newBody)
+
+    return @doc
+  end
+
+  private
+
+  def ensureParsedDocument()
+    if @doc == nil then
+      @doc = Document.new(@file)
+    end
+  end
+end
+
+=end