shalmaneser 1.2.0.rc4 → 1.2.rc5

data/lib/fred/FredConventions.rb

@@ -1,232 +0,0 @@
-# FredConventions
-# Katrin Erk June 05
-#
-# several small things that should be uniform
-# throughout the system
-
-require "common/ruby_class_extensions"
-
-require "common/EnduserMode"
-class Object
-
-###
-# joining and breaking up senses
-def fred_join_senses(senses)
-  return senses.sort().join("++")
-end
-
-def fred_split_sense(joined_senses)
-  return joined_senses.split("++")
-end
-
-###
-# fred_dirname
-#
-# constructs a directory name:
-# fred data directory / experiment ID / maindir / subdir
-#
-# if is_existing == existing, the directory is checked for existence,
-# if is_existing == new, it is created if necessary
-#
-# returns: a string
-def fred_dirname(exp,             # FredConfigData object
-                 maindir,         # string: main part of directory name
-                 subdir,          # string: subpart of directory name
-                 is_existing = "existing")  # string: "existing" or "new", default: existing
-
-  case is_existing
-  when "existing"
-    return File.existing_dir(exp.get("fred_directory"),
-                         exp.get("experiment_ID"),
-                         maindir,
-                         subdir)
-  when "new"
-    return File.new_dir(exp.get("fred_directory"),
-                         exp.get("experiment_ID"),
-                         maindir,
-                         subdir)
-  else
-    raise "Shouldn't be here: #{is_existing}"
-  end
-end
-
-####
-# filenames for feature files
-def fred_feature_filename(lemma, sense = nil, 
-			  do_binary = false)
-  if do_binary
-    return "fred.features.#{lemma}.SENSE.#{sense}"
-  else
-    return "fred.features.#{lemma}"
-  end
-end
-
-####
-# filenames for split files
-def fred_split_filename(lemma)
-  return "fred.split.#{lemma}"
-end
-
-###
-# deconstruct split filename
-# returns: lemma
-def deconstruct_fred_split_filename(filename)
-  basename = File.basename(filename)
-  if basename =~ /^fred\.split\.(.*)/
-    return $1
-  else
-    return nil
-  end
-end
-
-###
-# deconstruct feature file name
-# returns: hash with keys
-# "lemma"
-# "sense
-def deconstruct_fred_feature_filename(filename)
-
-  basename = File.basename(filename)
-  retv = Hash.new()
-  # binary: 
-  # fred.features.#{lemma}.SENSE.#{sense}
-  if basename =~ /^fred\.features\.(.*)\.SENSE\.(.*)$/
-    retv["lemma"] = $1
-    retv["sense"] = $2
-  elsif basename =~ /^fred\.features\.(.*)/
-    # fred.features.#{lemma}
-    retv["lemma"] = $1
-
-  else
-    # complete mismatch
-    return nil
-  end
-
-  return retv
-end
-
-####
-# filename for answer key files
-def fred_answerkey_filename(lemma)
-  return "fred.answerkey.#{lemma}"
-end
-
-###
-# classifier directory
-def fred_classifier_directory(exp,     # FredConfigData object
-                              splitID = nil) # string or nil
-
-  if exp.get("classifier_dir")
-    # user-specified classifier directory
-
-    if splitID
-      return File.new_dir(exp.get("classifier_dir"), splitID)
-    else
-      return File.new_dir(exp.get("classifier_dir"))
-    end
-
-  else
-    # my classifier directory
-    if splitID
-      return fred_dirname(exp, "classifiers", splitID, "new")
-    else
-      return fred_dirname(exp, "classifiers", "all", "new")
-    end
-  end
-end
-
-###
-# classifier file
-def fred_classifier_filename(classifier, lemma, sense=nil)
-  if sense
-    return "fred.classif.#{classifier}.LEMMA.#{lemma}.SENSE.#{sense}"
-  else
-    return "fred.classif.#{classifier}.LEMMA.#{lemma}"
-  end
-end
-
-def deconstruct_fred_classifier_filename(filename)
-  retv = Hash.new()
-  if filename =~ /^fred\.classif\.(.*)\.LEMMA\.(.*)\.SENSE\.(.*)$/
-    retv["lemma"] = $2
-    retv["sense"] = $3
-  elsif filename =~ /^fred\.classif\.(.*)\.LEMMA\.(.*)$/
-    retv["lemma"] = $2
-  end
-  return retv
-end
-
-###
-# result file
-def fred_result_filename(lemma)
-  return "fred.result.#{lemma.gsub(/\./, "_")}"
-end
-
-##########
-# lemma and POS: combine into string separated by 
-# a separator character
-#
-# fred_lemmapos_combine: take two strings, return combined string
-#      if POS is nil, returns lemma<separator character>
-# fred_lemmapos_separate: take one string, return two strings
-#      if no POS could be retrieved, returns nil as POS and the whole string as lemma
-def fred_lemmapos_combine(lemma, # string
-			  pos)   # string
-  return lemma.to_s + "." + pos.to_s.gsub(/\./, "DOT")
-end
-
-###
-def fred_lemmapos_separate(lemmapos)  # string
-  pieces = lemmapos.split(".")
-  if pieces.length() > 1
-	return [ pieces[0..-2].join("."), pieces[-1] ]
-  else
-    # no POS found, treat all of lemmapos as lemma
-    return [ lemmapos, nil ]
-  end
-end
-end
-
-########################################
-# given a SynNode object representing a terminal,
-# return:
-# - the word
-# - the lemma
-# - the part of speech
-# - the named entity (if any)
-#
-# as a tuple
-#
-# WARNING: word and lemma are turned to lowercase
-module WordLemmaPosNe
-  def word_lemma_pos_ne(syn_obj, # SynNode object
-                        i)       # SynInterpreter class
-    unless syn_obj.is_terminal?
-      $stderr.puts "Featurization warning: unexpectedly received non-terminal"
-      return [ nil, nil, nil, nil ]
-    end
-
-    word = syn_obj.word()
-    if word
-      word.downcase!
-    end
-
-    lemma = i.lemma_backoff(syn_obj)
-    if lemma and SalsaTigerXMLHelper.unescape(lemma) == "<unknown>"
-      lemma = nil
-    end
-    if lemma
-      lemma.downcase!
-    end
-
-    pos = syn_obj.part_of_speech()
-
-    ne = syn_obj.get_attribute("ne")
-    unless ne
-      ne = syn_obj.get_attribute("headof_ne")
-    end
-
-    return [word, lemma, pos, ne]
-  end
-end
-

data/lib/fred/FredDetermineTargets.rb

@@ -1,319 +0,0 @@
-require "fred/FileZipped"
-
-require "fred/fred_config_data"
-require "common/SynInterfaces"
-require "fred/FredConventions"
-
-
-########################################
-# target determination classes:
-# either determine targets from existing annotation
-# with frames,
-# or use all known targets.
-class Targets
-  attr_reader :targets_okay
-
-  ###
-  def initialize(exp,                 # experiment file object
-                 interpreter_class,   # SynInterpreter class, or nil
-                 mode)                # string: "r", "w", "a", as in files
-    @exp = exp
-    @interpreter_class = interpreter_class
-
-    # keep recorded targets here.
-    # try to read old list now.
-    @targets = Hash.new()
-
-    # write target info in the classifier directory.
-    # This is _not_ dependent on a potential split ID
-    @dir = File.new_dir(fred_classifier_directory(@exp), "targets")
-
-    @targets_okay = true
-    case mode
-    when "w"
-      # start from scratch, no list of targets
-    when "a", "r"
-      # read existing file containing targets
-      begin
-        file = FileZipped.new(@dir + "targets.txt.gz")
-      rescue
-        # no pickle present: signal this
-        @targets_okay = false
-        return
-      end
-      file.each { |line|
-        line.chomp!
-        if line =~ /^LEMMA (.+) SENSES (.+)$/
-          lemmapos = $1
-          senses = $2.split()
-          lemmapos.gsub!(/ /, '_')
-          #lemmapos.gsub!(/\.[A-Z]\./, '.')
-         @targets[lemmapos] = senses
-        end
-      }
-
-    else
-      $stderr.puts "Error: shouldn't be here."
-      exit 1
-    end
-
-    if ["w", "a"].include? mode
-      @record_targets = true
-    else
-      @record_targets = false
-    end
-  end
-
-  ###
-  # determine_targets:
-  # for a given SalsaTigerSentence,
-  # determine all targets,
-  # each as a _single_ main terminal node
-  #
-  # We need a single terminal node in order
-  # to compute the context window
-  #
-  # returns:
-  #  hash: target_IDs -> list of senses
-  #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
-  #  
-  #  where a sense is represented as a hash:
-  #  "sense": sense, a string
-  #  "obj":   FrameNode object
-  #  "all_targets": list of node IDs, may comprise more than a single node
-  #  "lex":   lemma, or multiword expression in canonical form
-  #  "sid": sentence ID
-  def determine_targets(sent)
-    raise "overwrite me"
-  end
-
-  ##
-  # returns a list of lemma-pos combined strings
-  def get_lemmas()
-    return @targets.keys()
-  end
-
-  ##
-  # access to lemmas and POS, returns a list of pairs [lemma, pos] (string*string)
-  def get_lemma_pos()
-
-    return @targets.keys().map { |lemmapos| fred_lemmapos_separate(lemmapos) }
-  end
-
-  ##
-  # access to senses
-  def get_senses(lemmapos) # string, result of fred_lemmapos_combine 
-    @targets[lemmapos] ? @targets[lemmapos] : []
-  end
-
-  ##
-  # write file
-  def done_reading_targets()
-    begin
-      file = FileZipped.new(@dir + "targets.txt.gz", "w")
-    rescue
-      $stderr.puts "Error: Could not write file #{@dir}targets.txt.gz"
-      exit 1
-    end
-
-    @targets.each_pair { |lemma, senses|
-      file.puts "LEMMA #{lemma} SENSES "+ senses.join(" ")
-    }
-
-    file.close
-  end
-
-  ###############################
-  protected
-  
-  ##
-  # record: record occurrence of a lemma/sense pair
-  # <@targets> data structure
-  def record(target_info) 
-    lemmapos = fred_lemmapos_combine(target_info["lex"], target_info["pos"])
-    unless @targets[lemmapos]
-      @targets[lemmapos] = []
-    end
- 
-    unless @targets[lemmapos].include? target_info["sense"]
-      @targets[lemmapos] << target_info["sense"]
-    end
-  end
-end
-
-########################################
-class FindTargetsFromFrames < Targets
-  ###
-  # determine_targets:
-  # use existing frames to find targets
-  #
-  # returns:
-  #  hash: target_IDs -> list of senses
-  #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
-  #  
-  #  where a sense is represented as a hash:
-  #  "sense": sense, a string
-  #  "obj":   FrameNode object
-  #  "all_targets": list of node IDs, may comprise more than a single node
-  #  "lex":   lemma, or multiword expression in canonical form
-  #  "sid": sentence ID
-  def determine_targets(st_sent) #SalsaTigerSentence object
-   retv = Hash.new()
-    st_sent.each_frame { |frame_obj|
-      # instance-specific computation:
-      # target and target positions
-      # WARNING: at this moment, we are 
-      # not considering true multiword targets for German.
-      # Remove the "no_mwe" parameter in main_node_of_expr
-      # to change this
-      term = nil
-      all_targets = nil
-      if frame_obj.target.nil? or frame_obj.target.children.empty?
-        # no target, nothing to record
-
-      elsif @exp.get("language") == "de"
-        # don't consider true multiword targets for German
-        all_targets = frame_obj.target.children()
-        term = @interpreter_class.main_node_of_expr(all_targets, "no_mwe")
-
-      else
-        # for all other languages: try to figure out the head target word
-        # anyway
-        all_targets = frame_obj.target.children()
-        term = @interpreter_class.main_node_of_expr(all_targets)
-      end
-
-      if term and term.is_splitword?
-        # don't use parts of a word as main node
-          term = term.parent()
-      end
-      if term and term.is_terminal?
-        key = [all_targets.map { |t| t.id() }, term.id()]
-
-        unless retv[key]
-          retv[key] = Array.new()
-        end
-
-        pos = frame_obj.target().get_attribute("pos")
-        # gold POS available, may be in wrong form,
-        # i.e. not the same strings that @interpreter_class.category()
-        # would return
-        case pos
-        when /^[Vv]$/
-          pos = "verb"
-        when /^[Nn]$/
-          pos = "noun"
-        when /^[Aa]$/
-          pos = "adj"
-        when nil
-          pos = @interpreter_class.category(term)
-        end
-
-        target_info = {
-          "sense" => frame_obj.name(),
-          "obj" => frame_obj,
-          "all_targets" => frame_obj.target.children().map { |ch| ch.id() },
-          "lex" => frame_obj.target().get_attribute("lemma"),
-          "pos" => pos,
-          "sid" => st_sent.id()
-        }
-      #print "lex ", frame_obj.target(), " und ",frame_obj.target().get_attribute("lemma"), "\n"
-        retv[key] << target_info
-        if @record_targets
-          record(target_info)
-        end
-      end
-    }
-    return retv
-  end
-end
-
-########################################
-class FindAllTargets < Targets
-  ###
-  # determine_targets:
-  # use all known lemmas, minus stopwords
-  def initialize(exp,
-                 interpreter_class)
-    # read target info from file
-    super(exp, interpreter_class, "r")
-    @training_lemmapos_pairs = get_lemma_pos()
-
-    get_senses(@training_lemmapos_pairs)
-    # list of words to exclude from assignment, for now
-    @stoplemmas = [
-                   "have", 
-                   "do", 
-                   "be"
-                   #      "make"
-                  ]
-  
-  end
-
-  ####
-  #
-  # returns:
-  #  hash: target_IDs -> list of senses
-  #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
-  #  
-  #  where a sense is represented as a hash:
-  #  "sense": sense, a string
-  #  "obj":   FrameNode object
-  #  "all_targets": list of node IDs, may comprise more than a single node
-  #  "lex":   lemma, or multiword expression in canonical form
-  #  "sid": sentence ID
-  def determine_targets(sent) #SalsaTigerSentence object
-    # map target IDs to list of senses, in our case always [ nil ]
-    # because we assume that the senses of the targets we point out
-    # are unknown
-    retv = Hash.new()
-    # iterate through terminals of the sentence, check for inclusion
-    # of their lemma in @training_lemmas
-    sent.each_terminal { |node|
-      # we know this lemma from the training data,
-      # and it is not an auxiliary,
-      # and it is not in the stopword list
-      # and the node does not represent a preposition
-
-      ### modified by ines, 17.10.2008
-      lemma = @interpreter_class.lemma_backoff(node)
-      pos = @interpreter_class.category(node)
-
-#	print "lemma ", lemma, " pos ", pos, "\n"
-#      reg = /\.[ANV]/
-#      if !reg.match(lemma) 
-#        if /verb/.match(pos) 
-#          lemma = lemma + ".V"
-#        elsif /noun/.match(pos) 
-#          lemma = lemma + ".N"
-#        elsif /adj/.match(pos) 
-#          lemma = lemma + ".A"
-#        end
-#        print "LEMMA ", lemma, " POS ", pos, "\n" 
-#      end
-
-      if (@training_lemmapos_pairs.include? [lemma, pos] and
-          not(@interpreter_class.auxiliary?(node)) and
-          not(@stoplemmas.include? lemma) and
-          not(pos == "prep"))
-          key = [ [ node.id() ], node.id() ]
-
-          # take this as a target.
-          retv[ key ] = [ 
-                         {
-                           "sense" => nil,
-                           "obj" => nil,
-                           "all_targets" => [ node.id() ],
-                           "lex" => lemma,
-                           "pos" => pos,
-                           "sid" => sent.id()
-                         } ]
-          # no recording of target info,
-          # since we haven't determined anything new
-        end
-    }
-
-    return retv
-  end
-end
-