shalmaneser-lib 1.2.rc5

data/lib/external_systems.rb

@@ -0,0 +1,251 @@
+# ExternalSystems.rb
+#
+# ke oct/nov 2005
+#
+# Store all known interfaces to
+# systems that do syntactic analysis
+#
+# Given the name of a system and the service that the
+# system performs, return the appropriate interface
+#
+# There are two types of interfaces to syntactic analysis systems:
+# - interfaces:
+#   offer methods for syntactic analysis,
+#   and the transformation to Salsa/Tiger XML and SalsaTigerSentence objects
+# - interpreters:
+#   interpret the resulting Salsa/Tiger XML (represented as
+#   SalsaTigerSentence and SynNode objects), e.g.
+#   generalize over part of speech;
+#   describe the path between a pair of nodes both as a path
+#   and (potentially) as a grammatical function of one of the nodes;
+#   determine whether a node describes a verb, and in which voice;
+#   determine the head of a constituent
+#
+# Abstract classes for both interfaces and interpreters
+# are in AbstractSynInterface.rb
+
+require "ruby_class_extensions"
+require 'logging'
+
+# The list of available interface packages
+# is at the end of this file.
+# Please enter additional interfaces there.
+
+# @todo AB: [2015-12-16 Wed 01:03]
+#   After decoupling in OptParser and ConfigData classes move this
+#   to Frappe.
+module Shalmaneser
+  class ExternalSystems
+    ###
+    # class variable:
+    # list of all known interface classes
+    # add to it using add_interface()
+    @interfaces = []
+
+    ###
+    # class variable:
+    # list of all known interpreter classes
+    # add to it using add_interpreter()
+    @interpreters = []
+
+    ###
+    # add interface/interpreter
+    def self.add_interface(class_name)
+      LOGGER.debug "Initializing interface <#{class_name}>."
+      @interfaces << class_name
+    end
+
+    def self.add_interpreter(class_name)
+      LOGGER.debug "Initializing interpreter <#{class_name}>."
+      @interpreters << class_name
+    end
+
+    ###
+    # check_interfaces_abort_if_missing:
+    #
+    # Given an experiment file, use some_system_missing? to
+    # determine whether the system can be run with the requested
+    # syntactic processing, exit with an error message if that is not possible
+    # @param [FrappeConfigData] exp Experiment description.
+    def self.check_interfaces_abort_if_missing(exp)
+      if (missing = some_system_missing?(exp))
+        interwhat, services = missing
+
+        $stderr.puts
+        $stderr.puts "ERROR: I am missing an #{interwhat} for "
+        services.each_pair { |service, system_name|
+          $stderr.puts "\tservice #{service}, system #{system_name}"
+        }
+        $stderr.puts
+        $stderr.puts "I have the following interfaces:"
+        @interfaces.each { |interface_class|
+          $stderr.puts "\tservice #{interface_class.service}, system #{interface_class.system}"
+        }
+        $stderr.puts "I have the following interpreters:"
+        @interpreters.each { |interpreter_class|
+          $stderr.print "\t"
+          $stderr.print interpreter_class.systems.to_a.map { |service, system_name|
+            "service #{service}, system #{system_name}"
+          }.join("; ")
+          unless interpreter_class.optional_systems.empty?
+            $stderr.print ", optional: "
+            $stderr.print interpreter_class.optional_systems.to_a.map { |service, system_name|
+              "service #{service}, system #{system_name}"
+            }.join("; ")
+          end
+          $stderr.puts
+        }
+        $stderr.puts
+        $stderr.puts "Please adapt your experiment file."
+        exit 1
+      end
+    end
+
+    ###
+    # given the name of a system and the service that it
+    # performs, find the matching interface class
+    #
+    # system: string: name of system, e.g. collins
+    # service: string: service, e.g. parser
+    #
+    # returns: SynInterface class
+    def self.get_interface(service, system)
+      interfaces = @interfaces.select do |interface_class|
+        interface_class.system == system && interface_class.service == service
+      end
+
+      unless interfaces.any?
+        raise "I've been requested an interface for #{service} and #{system}, "\
+              'but I cannot find any. Please correct your experiment files.'
+      end
+
+      # @todo AB: Actually it's bad logic, but no idea for now how to handle it.
+      interfaces.first
+    end
+
+    ###
+    # helper for get_interpreter:
+    def self.get_interpreter_according_to_exp(exp)
+      ExternalSystems.get_interpreter(ExternalSystems.requested_services(exp))
+    end
+
+    ###
+    # given the names and services of a set of systems,
+    # find the matching interpreter class
+    #
+    # an interpreter class has both obligatory systems
+    # (they need to be present for this class to apply)
+    # and optional systems (they may or may not be present
+    # for the class to apply, but no other system performing
+    # the same service may)
+    #
+    # systems:
+    # hash: service(string) -> system name(string)
+    #
+    # returns: SynInterpreter class
+    def self.get_interpreter(systems)
+      # try to find an interface class with the given
+      # service-name pairs
+
+      @interpreters.each { |interpreter_class|
+
+        if interpreter_class.systems.to_a.big_and { |service, system|
+             # all obligatory entries of interpreter_class
+             # are in systems
+             systems[service] == system
+           } and
+          interpreter_class.optional_systems.to_a.big_and { |service, system|
+             # all optional entries of interpreter_class are
+             # either in systems, or the service isn't in systems at all
+             systems[service].nil? or systems[service] == system
+           } and
+          systems.to_a.big_and { |service, system|
+             # all entries in names are in either
+             # the obligatory or optional set for interpreter_class
+             interpreter_class.systems[service] == system or
+               interpreter_class.optional_systems[service] == system
+           }
+          return interpreter_class
+        end
+      }
+
+      # at this point, detection of a suitable interpreter class has failed
+      return nil
+    end
+
+    ################
+    private
+
+    ###
+    # knows about possible services that can be set in
+    # the experiment file, and where the names of
+    # the matching systems will be found in the experiment file data structure
+    #
+    # WARNING: adapt this when you introduce new services!
+    #
+    # returns: a hash
+    #  <service> => system_name
+    #
+    #  such that for each service/system name pair:
+    #  the service with the given name has been requested in
+    #  the experiment file, and the names of the systems to be used
+    #  for performing the service
+    def self.requested_services(exp)
+      services = {}
+      [
+        {"flag" => "do_postag", "service" => "pos_tagger"},
+        {"flag" => "do_lemmatize", "service" => "lemmatizer"},
+        {"flag" => "do_parse", "service" => "parser"}
+      ].each do |hash|
+        # yes, perform this service
+        if exp.get(hash["flag"])
+          services[hash["service"]] = exp.get(hash["service"])
+        end
+      end
+
+      services
+    end
+
+    ###
+    # some_system_missing?
+    # returns nil if I have interfaces and interpreters
+    # for all services requested in the given experiment file
+    # else:
+    # returns pair [interface or interpreter, info]
+    #  where the 1st element is either 'interface' or 'interpreter',
+    #  and the 2nd element is a hash mapping services to system names:
+    #  the services that could not be provided
+    # @param [FrappeConfigdata] exp FrappeConfigData object to check all the systems.
+    def self.some_system_missing?(exp)
+      missing_systems = nil
+      # check interfaces
+      requested_services(exp).each_pair do |service, system_name|
+        unless get_interface(service, system_name)
+          missing_systems = ["interface", {service => system_name}]
+        end
+      end
+
+      # check interpreter
+      unless get_interpreter_according_to_exp(exp)
+        missing_systems = ["interpreter", services]
+      end
+
+      # everything okay
+      missing_systems
+    end
+  end
+end
+
+# @todo AB: We should require programmatically all files in
+# <frappe/interpreters> and <frappe/interfaces>.
+require 'frappe/interfaces/collins_interface'
+require 'frappe/interpreters/collins_treetagger_interpreter'
+require 'frappe/interpreters/collins_tnt_interpreter'
+require 'frappe/interfaces/berkeley_interface'
+require 'frappe/interpreters/berkeley_interpreter'
+require 'frappe/interfaces/stanford_interface'
+require 'frappe/interpreters/stanford_interpreter'
+require 'frappe/interfaces/treetagger_interface'
+require 'frappe/interfaces/treetagger_pos_interface'
+require 'frappe/interpreters/treetagger_interpreter'
+require 'frappe/interpreters/empty_interpreter'

data/lib/framenet_format/fn_corpus_aset.rb

@@ -0,0 +1,209 @@
+#####################
+# class to keep data for one frame
+class FNCorpusAset
+  attr_reader :layers, :aset_type, :aset_id, :frame_name, :lu
+
+  #######
+  # Analyze RegXML object, store in object variables:
+  #
+  # @aset_type: "frame" or "NER"
+  # @frame_name: frame name for "frame" type
+  # @lu: LU for "frame" type
+  # @aset_id: ID of the annotation set
+  # @layers: hash: layer type (FE, GF, PT, Target, NER) -> [offset, "start"/"stop"]  -> list of labels
+  #     string -> int*string -> array:string
+  #
+  def initialize(aset, #RegXML object
+                 charidx) # array of pairs [start index, stop index] int*int
+
+    @layers = {}
+    @frame_name = nil
+    @lu = nil
+    @aset_type = nil
+
+    attributes = aset.attributes
+
+    @aset_id = attributes["ID"]
+
+    if attributes["frameName"]
+      # all of these seem to be frames. store in 'frames' array
+      unless attributes["luName"]
+        $stderr.puts "FNCorpusAset warning: cannot determine LU name"
+        $stder.puts aset.to_s
+        return
+      end
+      @aset_type = "frame"
+      @frame_name = attributes["frameName"]
+      @lu = attributes["luName"]
+
+      unless (layers = aset.first_child_matching("layers"))
+        $stderr.puts "FNCorpusAset warning: unexpectedly no layers found"
+        $stderr.puts aset.to_s
+        return
+      end
+
+      layers.each_child_matching("layer") { |l| analyze_layer(l, charidx) }
+
+    else
+      # all we seem to get here are named entity labels.
+      @aset_type = "NER"
+
+      unless (layers = aset.first_child_matching("layers"))
+        $stderr.puts "FNCorpusAset Warning: unexpectedly no layers found"
+        $stderr.puts aset.to_s
+        return
+      end
+      unless (layer = layers.first_child_matching("layer"))
+        $stderr.puts "FNCorpusAset Warning: unexpectedly no layers found"
+        $stderr.puts aset.to_s
+        return
+      end
+
+      unless layer.attributes["name"] == "NER"
+        $stderr.puts "FNCorpusAset Warning: unexpected layer #{layer.attributes["name"]}, was expecting only an NER layer."
+        $stderr.puts aset.to_s
+        return
+      end
+
+      analyze_layer(layer, charidx)
+
+    end
+  end
+
+
+  #############
+  # input: <layer> RegXML object
+  # analyze this, put into @layers data structure
+  def analyze_layer(layer, # RegXML object
+                    charidx) # array:int*int pairs start/end index of words
+    layer_name = layer.attributes["name"]
+    unless layer_name
+      $stderr.puts "FNCorpusAset warning: cannot determine layer name"
+      $stderr.puts layer.to_s
+      return
+    end
+
+    # FN-specific: skip 2nd layer FEs for now
+    if layer_name == "FE" and layer.attributes["rank"] == "2"
+      return
+    end
+
+    unless @layers[layer_name]
+      @layers[layer_name] = {}
+    end
+
+    unless (labels = layer.first_child_matching("labels"))
+      # nothing to record for this layer
+      return
+    end
+
+
+    # taking over much of analyse_layer from class FrameXML
+    thisLayer = []
+
+    labels.each_child_matching("label") { |label|
+      attributes = label.attributes
+      if attributes["itype"] =~ /NI/
+        # null instantiation, ignore
+        next
+      end
+
+      if not(attributes["start"]) and not(attributes["end"])
+        # no start and end labels
+        next
+      end
+      thisLayer << [attributes["name"], attributes["start"].to_i, attributes["end"].to_i]
+    }
+
+    # sanity check: do indices
+    # match word start and end indices?
+    thisLayer = verify_annotation(thisLayer, charidx)
+
+    # sanity check: verify that
+    # we don't have overlapping labels
+
+    deleteHash = {} # keep track of the labels which are to be deleted
+    # i -> Boolean
+
+    thisLayer.each_index {|i|
+      # efficiency: skip already delete labels
+      if deleteHash[i]
+        next
+      end
+      this_label, this_from , this_to = thisLayer[i]
+
+      # compare with all remaining labels
+      (i+1..thisLayer.length-1).to_a.each { |other_i|
+        other_label,other_from,other_to = thisLayer[other_i]
+
+        # overlap? Throw out the later FE
+        if this_from <= other_from and other_from <= this_to
+          $stderr.puts "Warning: Label overlap, deleting #{other_label}"
+          deleteHash[other_i] = true
+        elsif this_from <= other_to and other_to <= this_to
+          $stderr.puts "Warning: Label overlap, deleting #{this_label}"
+          delete_hash[i] = true
+        end
+      }
+      # matched with all other labels. If "keep", return
+
+      if deleteHash[i]
+      # $stderr.puts " deleting entry #{i}"
+      else
+        [ [this_from, "start"], [this_to, "stop"]].each { |offset, start_or_stop|
+          unless @layers[layer_name].has_key?([offset, start_or_stop])
+            @layers[layer_name][[offset, start_or_stop]] = []
+          end
+          @layers[layer_name][ [offset, start_or_stop] ] << this_label
+        }
+      end
+    }
+  end
+
+  ##############3
+  # verify found triples label/from_index/to_index
+  # against given start/end indices of words
+  #
+  # returns: triples, possibly changed
+  def verify_annotation(found,        # array: label/from/to, string*int*int
+                        charidx)      # array: from/to, int*int
+
+    return found.map {|element, start, stop|
+
+      newstart = start
+      newstop = stop
+
+      # compare against word start/stop indices
+      charidx.each_index{|j|
+        unless j== 0
+          pstartidx, pstopidx = charidx[j-1]
+        end
+        startidx, stopidx = charidx[j]
+
+        if (start > startidx and start <= stopidx) or
+          (j != 0 and start > pstopidx and start < startidx)
+          newstart = startidx
+        end
+
+        if (stop >= startidx and stop < stopidx)
+          newstop = stopidx
+        elsif (j != 0 and stop > pstopidx and stop < startidx)
+          newstop = pstopidx
+        end
+      }
+
+      # change?
+      if start != newstart or stop != newstop
+        # report change
+        $stderr.puts "FNCorpusXML warning: Heuristics has changed element "+element
+        $stderr.puts "\tfrom ["+[start,stop].join(",")+"] to ["+[newstart,newstop].join(",")+"]"
+
+        [element, newstart, newstop]
+
+      else
+
+        [element, start, stop]
+      end
+    }
+  end
+end