frprep 0.0.1.prealpha

data/lib/fred/FredConfigData.rb

@@ -0,0 +1,182 @@
+# FredConfigData
+# Katrin Erk April 05
+#
+# Frame disambiguation system: 
+# access to a configuration and experiment description file
+
+require "common/ConfigData"
+
+##############################
+# Class FredConfigData
+#
+# inherits from ConfigData,
+# sets variable names appropriate to WSD task
+
+class FredConfigData < ConfigData
+  def initialize(filename)
+
+    # initialize config data object
+    super(filename,          # config file
+	  { 
+            "experiment_ID" => "string", # experiment ID
+            "enduser_mode" => "bool", # work in enduser mode? (disallowing many things)
+	    
+            "preproc_descr_file_train" => "string", # path to preprocessing files
+            "preproc_descr_file_test" => "string",
+            "directory_output" => "string", # path to Salsa/Tiger XML output directory
+
+            "verbose" => "bool" ,     # print diagnostic messages?
+            "apply_to_all_known_targets" => "bool", # apply to all known targets rather than the ones with a frame?
+           
+            "fred_directory" => "string",# directory for internal info
+            "classifier_dir" => "string", # write classifiers here
+
+            "classifier" => "list",  # classifiers
+
+            "dbtype" => "string",    # "mysql" or "sqlite"
+           
+            "host" => "string",      # DB access: sqlite only
+            "user" => "string",
+            "passwd" => "string",
+            "dbname" => "string",
+
+            # featurization info
+            "feature" => "list",     # which features to use for the classifier?
+            "binary_classifiers" => "bool",# make binary rather than n-ary clasifiers?
+	    "negsense" => "string",  # binary classifier: negative sense is..?
+            "numerical_features" => "string", # do what with numerical features?
+
+            # what to do with items that have multiple senses?
+            # 'binarize': binary classifiers, and consider positive
+            #          if the sense is among the gold senses
+            # 'join' : make one joint sense
+            # 'repeat' : make multiple occurrences of the item, one sense per occ
+            # 'keep' : keep as separate labels
+            #
+            # multilabel: consider as assigned all labels
+            # above a certain confidence threshold?
+            "handle_multilabel" => "string",
+            "assignment_confidence_threshold" => "float",
+            
+            # single-sentence context?
+            "single_sent_context" => "bool",
+
+            # noncontiguous input? then we need access to a larger corpus
+            "noncontiguous_input" => "bool",
+            "larger_corpus_dir" => "string",
+            "larger_corpus_format" => "string", 
+            "larger_corpus_encoding" => "string"
+	  },
+	  [ # variables
+            "train", 
+           "exp_ID"
+	  ]
+	  )
+
+    # set access functions for list features
+    set_list_feature_access("classifier",
+                            method("access_classifier"))
+    set_list_feature_access("feature",
+                            method("access_feature"))
+  end
+
+  ###
+  # protected
+
+  #####
+  # access_feature
+  # 
+  # access function for feature 'feature'
+  #
+  # assumed format:
+  #
+  #   feature = context 50
+  #   feature = context 2
+  #   feature = syn
+  #
+  # i.e. first the name of the feature type to use, then
+  # optionally a parameter,
+  # and the same feature can occur more than once (which makes sense
+  # only in case of parameters)
+  #
+  #
+  # returns: 
+  #  - If a feature is given as a parameter, 
+  #    - If the feature is not set in the experiment file, nil
+  #    - If the feature is set and has a parameter, the list of 
+  #      parameter values set for it. It is assumed that the parameters
+  #      are integers, and they are returned as integers
+  #    - If the feature is set and has no parameter, true
+  # - If no feature is given as parameter:
+  #   a list of all features that have been set in the experiment file
+  #   Each feature is given as a tuple: the first element is the feature (a string),
+  #   all further elements are options (integers)
+  def access_feature(val_list, # array:array:string: list of tuples defined in config file
+		               # for feature 'feature'
+		     feature=nil)  # string: feature type name
+
+    if feature
+      # access options for this feature
+
+      # get the right tuples
+      positives = val_list.select { |entries|
+        entries.first() == feature
+      }.map { |entries|
+        entries[1]
+      }
+      
+      if positives.empty?
+        # feature not defined
+        return nil
+        
+      elsif positives.compact().empty?
+        # feature defined, but no parameters
+        return true
+        
+      else
+        # feature defined, and has values
+        return positives.map { |par| par.to_i() }
+      end
+
+    else
+      # return all features that have been set
+      return val_list.map { |feature_name, *options|
+        [feature_name] + options.map { |o| o.to_i() }        
+      }
+    end
+  end
+
+  #####
+  # access_classifier
+  #
+  # access function for feature 'classifier'
+  #
+  # assumed format in the config file:
+  #
+  #   feature = path [option]*
+  #
+  # i.e. first the name of the feature type to use, then
+  # optionally options associated with that feature,
+  # e.g. 'argrec': use that feature only when computing argrec
+  #
+  # the access function is called with parameter val_list, an array of
+  # string tuples, one string tuple for each feature defined.
+  # the first string in the tuple is the feature name, the rest are the options
+  #
+  # returns: a list of pairs [feature_name(string), options(array:string)]
+  # of defined features
+  def access_classifier(val_list) # array:array:string: list of tuples defined in config file
+		               # for feature 'feature'
+    if val_list.nil?
+      return []
+    else
+      return val_list.map { |cl_descr_tuple|
+        [cl_descr_tuple.first, cl_descr_tuple[1..-1]]
+      }    
+    end
+  end
+
+end
+
+
+ 

data/lib/fred/FredConventions.rb

@@ -0,0 +1,232 @@
+# FredConventions
+# Katrin Erk June 05
+#
+# several small things that should be uniform
+# throughout the system
+
+require "common/ruby_class_extensions"
+
+require "common/EnduserMode"
+class Object
+
+###
+# joining and breaking up senses
+def fred_join_senses(senses)
+  return senses.sort().join("++")
+end
+
+def fred_split_sense(joined_senses)
+  return joined_senses.split("++")
+end
+
+###
+# fred_dirname
+#
+# constructs a directory name:
+# fred data directory / experiment ID / maindir / subdir
+#
+# if is_existing == existing, the directory is checked for existence,
+# if is_existing == new, it is created if necessary
+#
+# returns: a string
+def fred_dirname(exp,             # FredConfigData object
+                 maindir,         # string: main part of directory name
+                 subdir,          # string: subpart of directory name
+                 is_existing = "existing")  # string: "existing" or "new", default: existing
+
+  case is_existing
+  when "existing"
+    return File.existing_dir(exp.get("fred_directory"),
+                         exp.get("experiment_ID"),
+                         maindir,
+                         subdir)
+  when "new"
+    return File.new_dir(exp.get("fred_directory"),
+                         exp.get("experiment_ID"),
+                         maindir,
+                         subdir)
+  else
+    raise "Shouldn't be here: #{is_existing}"
+  end
+end
+
+####
+# filenames for feature files
+def fred_feature_filename(lemma, sense = nil, 
+			  do_binary = false)
+  if do_binary
+    return "fred.features.#{lemma}.SENSE.#{sense}"
+  else
+    return "fred.features.#{lemma}"
+  end
+end
+
+####
+# filenames for split files
+def fred_split_filename(lemma)
+  return "fred.split.#{lemma}"
+end
+
+###
+# deconstruct split filename
+# returns: lemma
+def deconstruct_fred_split_filename(filename)
+  basename = File.basename(filename)
+  if basename =~ /^fred\.split\.(.*)/
+    return $1
+  else
+    return nil
+  end
+end
+
+###
+# deconstruct feature file name
+# returns: hash with keys
+# "lemma"
+# "sense
+def deconstruct_fred_feature_filename(filename)
+
+  basename = File.basename(filename)
+  retv = Hash.new()
+  # binary: 
+  # fred.features.#{lemma}.SENSE.#{sense}
+  if basename =~ /^fred\.features\.(.*)\.SENSE\.(.*)$/
+    retv["lemma"] = $1
+    retv["sense"] = $2
+  elsif basename =~ /^fred\.features\.(.*)/
+    # fred.features.#{lemma}
+    retv["lemma"] = $1
+
+  else
+    # complete mismatch
+    return nil
+  end
+
+  return retv
+end
+
+####
+# filename for answer key files
+def fred_answerkey_filename(lemma)
+  return "fred.answerkey.#{lemma}"
+end
+
+###
+# classifier directory
+def fred_classifier_directory(exp,     # FredConfigData object
+                              splitID = nil) # string or nil
+
+  if exp.get("classifier_dir")
+    # user-specified classifier directory
+
+    if splitID
+      return File.new_dir(exp.get("classifier_dir"), splitID)
+    else
+      return File.new_dir(exp.get("classifier_dir"))
+    end
+
+  else
+    # my classifier directory
+    if splitID
+      return fred_dirname(exp, "classifiers", splitID, "new")
+    else
+      return fred_dirname(exp, "classifiers", "all", "new")
+    end
+  end
+end
+
+###
+# classifier file
+def fred_classifier_filename(classifier, lemma, sense=nil)
+  if sense
+    return "fred.classif.#{classifier}.LEMMA.#{lemma}.SENSE.#{sense}"
+  else
+    return "fred.classif.#{classifier}.LEMMA.#{lemma}"
+  end
+end
+
+def deconstruct_fred_classifier_filename(filename)
+  retv = Hash.new()
+  if filename =~ /^fred\.classif\.(.*)\.LEMMA\.(.*)\.SENSE\.(.*)$/
+    retv["lemma"] = $2
+    retv["sense"] = $3
+  elsif filename =~ /^fred\.classif\.(.*)\.LEMMA\.(.*)$/
+    retv["lemma"] = $2
+  end
+  return retv
+end
+
+###
+# result file
+def fred_result_filename(lemma)
+  return "fred.result.#{lemma.gsub(/\./, "_")}"
+end
+
+##########
+# lemma and POS: combine into string separated by 
+# a separator character
+#
+# fred_lemmapos_combine: take two strings, return combined string
+#      if POS is nil, returns lemma<separator character>
+# fred_lemmapos_separate: take one string, return two strings
+#      if no POS could be retrieved, returns nil as POS and the whole string as lemma
+def fred_lemmapos_combine(lemma, # string
+			  pos)   # string
+  return lemma.to_s + "." + pos.to_s.gsub(/\./, "DOT")
+end
+
+###
+def fred_lemmapos_separate(lemmapos)  # string
+  pieces = lemmapos.split(".")
+  if pieces.length() > 1
+	return [ pieces[0..-2].join("."), pieces[-1] ]
+  else
+    # no POS found, treat all of lemmapos as lemma
+    return [ lemmapos, nil ]
+  end
+end
+end
+
+########################################
+# given a SynNode object representing a terminal,
+# return:
+# - the word
+# - the lemma
+# - the part of speech
+# - the named entity (if any)
+#
+# as a tuple
+#
+# WARNING: word and lemma are turned to lowercase
+module WordLemmaPosNe
+  def word_lemma_pos_ne(syn_obj, # SynNode object
+                        i)       # SynInterpreter class
+    unless syn_obj.is_terminal?
+      $stderr.puts "Featurization warning: unexpectedly received non-terminal"
+      return [ nil, nil, nil, nil ]
+    end
+
+    word = syn_obj.word()
+    if word
+      word.downcase!
+    end
+
+    lemma = i.lemma_backoff(syn_obj)
+    if lemma and SalsaTigerXMLHelper.unescape(lemma) == "<unknown>"
+      lemma = nil
+    end
+    if lemma
+      lemma.downcase!
+    end
+
+    pos = syn_obj.part_of_speech()
+
+    ne = syn_obj.get_attribute("ne")
+    unless ne
+      ne = syn_obj.get_attribute("headof_ne")
+    end
+
+    return [word, lemma, pos, ne]
+  end
+end
+

data/lib/fred/FredDetermineTargets.rb

@@ -0,0 +1,324 @@
+require "fred/FileZipped"
+
+require "fred/FredConfigData"
+require "common/SynInterfaces"
+require "fred/FredConventions"
+
+
+########################################
+# target determination classes:
+# either determine targets from existing annotation
+# with frames,
+# or use all known targets.
+class Targets
+  attr_reader :targets_okay
+
+  ###
+  def initialize(exp,                 # experiment file object
+                 interpreter_class,   # SynInterpreter class, or nil
+                 mode)                # string: "r", "w", "a", as in files
+    @exp = exp
+    @interpreter_class = interpreter_class
+
+    # keep recorded targets here.
+    # try to read old list now.
+    @targets = Hash.new()
+
+    # write target info in the classifier directory.
+    # This is _not_ dependent on a potential split ID
+    @dir = File.new_dir(fred_classifier_directory(@exp), "targets")
+
+    @targets_okay = true
+    case mode
+    when "w"
+      # start from scratch, no list of targets
+    when "a", "r"
+      # read existing file containing targets
+      begin
+        file = FileZipped.new(@dir + "targets.txt.gz")
+      rescue
+        # no pickle present: signal this
+        @targets_okay = false
+        return
+      end
+      file.each { |line|
+        line.chomp!
+        if line =~ /^LEMMA (.+) SENSES (.+)$/
+          lemmapos = $1
+          senses = $2.split()
+          lemmapos.gsub!(/ /, '_')
+          #lemmapos.gsub!(/\.[A-Z]\./, '.')
+         @targets[lemmapos] = senses
+        end
+      }
+
+    else
+      $stderr.puts "Error: shouldn't be here."
+      exit 1
+    end
+
+    if ["w", "a"].include? mode
+      @record_targets = true
+    else
+      @record_targets = false
+    end
+  end
+
+  ###
+  # determine_targets:
+  # for a given SalsaTigerSentence,
+  # determine all targets,
+  # each as a _single_ main terminal node
+  #
+  # We need a single terminal node in order
+  # to compute the context window
+  #
+  # returns:
+  #  hash: target_IDs -> list of senses
+  #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
+  #  
+  #  where a sense is represented as a hash:
+  #  "sense": sense, a string
+  #  "obj":   FrameNode object
+  #  "all_targets": list of node IDs, may comprise more than a single node
+  #  "lex":   lemma, or multiword expression in canonical form
+  #  "sid": sentence ID
+  def determine_targets(sent)
+    raise "overwrite me"
+  end
+
+  ##
+  # returns a list of lemma-pos combined strings
+  def get_lemmas()
+    return @targets.keys()
+  end
+
+  ##
+  # access to lemmas and POS, returns a list of pairs [lemma, pos] (string*string)
+  def get_lemma_pos()
+
+    return @targets.keys().map { |lemmapos| fred_lemmapos_separate(lemmapos) }
+  end
+
+  ##
+  # access to senses
+  def get_senses(lemmapos) # string, result of fred_lemmapos_combine 
+
+    if @targets[lemmapos]
+      return @targets[lemmapos]
+    else
+      return []
+    end
+  end
+
+  ##
+  # write file
+  def done_reading_targets()
+    begin
+      file = FileZipped.new(@dir + "targets.txt.gz", "w")
+    rescue
+      $stderr.puts "Error: Could not write file #{@dir}targets.txt.gz"
+      exit 1
+    end
+
+    @targets.each_pair { |lemma, senses|
+      file.puts "LEMMA #{lemma} SENSES "+ senses.join(" ")
+    }
+
+    file.close()
+  end
+
+  ###############################
+  protected
+  
+  ##
+  # record: record occurrence of a lemma/sense pair
+  # @targets data structure
+  def record(target_info) 
+    lemmapos = fred_lemmapos_combine(target_info["lex"], target_info["pos"])
+    unless @targets[lemmapos]
+      @targets[lemmapos] = Array.new
+    end
+ 
+    unless @targets[lemmapos].include? target_info["sense"]
+      @targets[lemmapos] << target_info["sense"]
+    end
+  end
+end
+
+########################################
+class FindTargetsFromFrames < Targets
+  ###
+  # determine_targets:
+  # use existing frames to find targets
+  #
+  # returns:
+  #  hash: target_IDs -> list of senses
+  #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
+  #  
+  #  where a sense is represented as a hash:
+  #  "sense": sense, a string
+  #  "obj":   FrameNode object
+  #  "all_targets": list of node IDs, may comprise more than a single node
+  #  "lex":   lemma, or multiword expression in canonical form
+  #  "sid": sentence ID
+  def determine_targets(st_sent) #SalsaTigerSentence object
+   retv = Hash.new()
+    st_sent.each_frame { |frame_obj|
+      # instance-specific computation:
+      # target and target positions
+      # WARNING: at this moment, we are 
+      # not considering true multiword targets for German.
+      # Remove the "no_mwe" parameter in main_node_of_expr
+      # to change this
+      term = nil
+      all_targets = nil
+      if frame_obj.target.nil? or frame_obj.target.children.empty?
+        # no target, nothing to record
+
+      elsif @exp.get("language") == "de"
+        # don't consider true multiword targets for German
+        all_targets = frame_obj.target.children()
+        term = @interpreter_class.main_node_of_expr(all_targets, "no_mwe")
+
+      else
+        # for all other languages: try to figure out the head target word
+        # anyway
+        all_targets = frame_obj.target.children()
+        term = @interpreter_class.main_node_of_expr(all_targets)
+      end
+
+      if term and term.is_splitword?
+        # don't use parts of a word as main node
+          term = term.parent()
+      end
+      if term and term.is_terminal?
+        key = [all_targets.map { |t| t.id() }, term.id()]
+
+        unless retv[key]
+          retv[key] = Array.new()
+        end
+
+        pos = frame_obj.target().get_attribute("pos")
+        # gold POS available, may be in wrong form,
+        # i.e. not the same strings that @interpreter_class.category()
+        # would return
+        case pos
+        when /^[Vv]$/
+          pos = "verb"
+        when /^[Nn]$/
+          pos = "noun"
+        when /^[Aa]$/
+          pos = "adj"
+        when nil
+          pos = @interpreter_class.category(term)
+        end
+
+        target_info = {
+          "sense" => frame_obj.name(),
+          "obj" => frame_obj,
+          "all_targets" => frame_obj.target.children().map { |ch| ch.id() },
+          "lex" => frame_obj.target().get_attribute("lemma"),
+          "pos" => pos,
+          "sid" => st_sent.id()
+        }
+      #print "lex ", frame_obj.target(), " und ",frame_obj.target().get_attribute("lemma"), "\n"
+        retv[key] << target_info
+        if @record_targets
+          record(target_info)
+        end
+      end
+    }
+    return retv
+  end
+end
+
+########################################
+class FindAllTargets < Targets
+  ###
+  # determine_targets:
+  # use all known lemmas, minus stopwords
+  def initialize(exp,
+                 interpreter_class)
+    # read target info from file
+    super(exp, interpreter_class, "r")
+    @training_lemmapos_pairs = get_lemma_pos()
+
+    get_senses(@training_lemmapos_pairs)
+    # list of words to exclude from assignment, for now
+    @stoplemmas = [
+                   "have", 
+                   "do", 
+                   "be"
+                   #      "make"
+                  ]
+  
+  end
+
+  ####
+  #
+  # returns:
+  #  hash: target_IDs -> list of senses
+  #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
+  #  
+  #  where a sense is represented as a hash:
+  #  "sense": sense, a string
+  #  "obj":   FrameNode object
+  #  "all_targets": list of node IDs, may comprise more than a single node
+  #  "lex":   lemma, or multiword expression in canonical form
+  #  "sid": sentence ID
+  def determine_targets(sent) #SalsaTigerSentence object
+    # map target IDs to list of senses, in our case always [ nil ]
+    # because we assume that the senses of the targets we point out
+    # are unknown
+    retv = Hash.new()
+    # iterate through terminals of the sentence, check for inclusion
+    # of their lemma in @training_lemmas
+    sent.each_terminal { |node|
+      # we know this lemma from the training data,
+      # and it is not an auxiliary,
+      # and it is not in the stopword list
+      # and the node does not represent a preposition
+
+      ### modified by ines, 17.10.2008
+      lemma = @interpreter_class.lemma_backoff(node)
+      pos = @interpreter_class.category(node)
+
+#	print "lemma ", lemma, " pos ", pos, "\n"
+#      reg = /\.[ANV]/
+#      if !reg.match(lemma) 
+#        if /verb/.match(pos) 
+#          lemma = lemma + ".V"
+#        elsif /noun/.match(pos) 
+#          lemma = lemma + ".N"
+#        elsif /adj/.match(pos) 
+#          lemma = lemma + ".A"
+#        end
+#        print "LEMMA ", lemma, " POS ", pos, "\n" 
+#      end
+
+      if (@training_lemmapos_pairs.include? [lemma, pos] and
+          not(@interpreter_class.auxiliary?(node)) and
+          not(@stoplemmas.include? lemma) and
+          not(pos == "prep"))
+          key = [ [ node.id() ], node.id() ]
+
+          # take this as a target.
+          retv[ key ] = [ 
+                         {
+                           "sense" => nil,
+                           "obj" => nil,
+                           "all_targets" => [ node.id() ],
+                           "lex" => lemma,
+                           "pos" => pos,
+                           "sid" => sent.id()
+                         } ]
+          # no recording of target info,
+          # since we haven't determined anything new
+        end
+    }
+
+    return retv
+  end
+end
+