shalmaneser-fred 1.2.0.rc4 → 1.2.rc5

data/lib/fred/FredFeaturize.rb

@@ -1,602 +0,0 @@
-# FredFeaturize
-#
-# Featurization for WSD
-#
-# Katrin Erk April 2007
-#
-# feature types currently allowed:
-# - context (with parameter giving context size; may be set several times)
-# - syntax
-# - synsem
-#
-# features in Meta-feature file:
-#
-# CX: context: word/lemma/pos/ne
-# CH: children: grfunc/word/lemma/pos/ne
-# PA: parents: grfunc/word/lemma/pos/ne
-# SI: sibling: parent/grfunc/word/lemma/pos/ne
-# TA: target: word/lemma/pos/ne
-
-
-
-require "delegate"
-
-#######
-
-require "fred/FileZipped"
-require "common/Parser"
-require "common/RegXML"
-require "common/SalsaTigerRegXML"
-require "common/SalsaTigerXMLHelper"
-
-require "fred/fred_config_data"
-require "fred/FredConventions"
-require "common/prep_helper"
-require "common/SynInterfaces"
-
-require "fred/FredBOWContext"
-require "fred/FredDetermineTargets"
-require "fred/FredFeatures"
-
-####################################
-# grammatical function computation:
-# given a sentence, keep all grammatical function relations in a hash
-# for faster access
-class GrammaticalFunctionAccess
-
-  def initialize(interpreter_class)
-    @interpreter_class = interpreter_class
-    @to = Hash.new( [] ) # default: return empty array
-    @from = Hash.new( [] ) # default: return empty array
-  end
-
-  def set_sent(sent) # SalsaTigerRegXML sentence
-
-    @to.clear()
-    @from.clear()
-
-    sent.each_syn_node { |current|
-
-      current_head = @interpreter_class.head_terminal(current)
-      unless current_head
-        next
-      end
-
-      @interpreter_class.gfs(current, sent).map { |rel, node|
-        # PPs: use head noun rather than preposition as head
-        # Sbar, VP: use verb 
-        if (n = @interpreter_class.informative_content_node(node))
-          [rel, n]
-        else
-          [rel, node]
-        end
-      }.each { |rel, node|
-
-        rel_head = @interpreter_class.head_terminal(node)
-        unless rel_head
-          next
-        end
-
-        unless @to.has_key? current_head
-          @to[current_head] = Array.new()
-        end
-        unless @to[current_head].include? [rel, rel_head]
-          @to[current_head] << [rel, rel_head]
-        end
-
-        unless @from.has_key? rel_head
-          @from[rel_head] = Array.new()
-        end
-        unless @from[rel_head].include? [rel, current_head]
-          @from[rel_head] << [rel, current_head]
-        end
-      }
-    }
-    #     $stderr.puts "Changed sentence:"
-    #     @to.each_pair { |p, ch|
-    #       $stderr.puts "\t#{p.id()}: " + ch.map { |rel, n| rel + "/"+n.id()}.join(", ")
-    #     }
-    #     $stdin.gets()
-  end
-
-  def get_children(node)
-    return @to[node]
-  end
-
-  def get_parents(node)
-    return @from[node]
-  end
-end
-
-####################################
-# main class of this package
-####################################
-class FredFeaturize < DelegateClass(GrammaticalFunctionAccess)
-
-  include WordLemmaPosNe
-
-  #####
-  def initialize(exp_obj, # FredConfigData object
-		 options, # hash: runtime option name (string) => value(string)
-                 varhash = {}) # optional parameter: "refeaturize"
-
-    ##
-    # evaluate runtime options
-    if $ENDUSER_MODE
-      # only possible dataset: test
-      @dataset = "test"
-    else
-      @dataset = nil
-    end
-    @append_rather_than_overwrite = false
-
-
-    options.each_pair do |opt, arg|
-      case opt
-      when '--dataset'
-	@dataset = arg
-	unless ["train", "test"].include? @dataset
-	  $stderr.puts "--dataset needs to be either 'train' or 'test'"
-          exit 1
-	end
-
-      when '--append'
-        @append_rather_than_overwrite = true
-
-      else
-	# case of unknown arguments has been dealt with by fred.rb
-      end
-    end
-
-    # further sanity checks
-     if @dataset.nil?
-      $stderr.puts  "Please set --dataset: one of 'train', 'test'"
-       exit 1
-    end
-
-    in_enduser_mode_ensure(@dataset == "test")
-
-    # evaluate optional "refeaturize" argument
-    # "refeaturize": reuse meta-feature set,
-    # just redo CSV featurization
-    if varhash["refeaturize"]
-      @refeaturize = varhash["refeaturize"]
-    else
-      @refeaturize = false
-    end
-
-    # prepare experiment file: add preprocessing experiment file data
-    @exp = exp_obj
-
-    # @note AB: The following is desabled because we don't want to use
-    #   the dependence on {PrepConfigData}. We duplicate options:
-    #   <do_postag>, <pos_tagger>, <do_lemmatize>, <lemmatizer>,
-    #   <do_parse>, <parser>, <directory_preprocessed>
-    #   in the experiment file of Fred.
-    #
-    # preproc_expname = @exp.get("preproc_descr_file_" + @dataset)
-    # if not(preproc_expname)
-    #   $stderr.puts "Please set the name of the preprocessing exp. file name"
-    #   $stderr.puts "in the experiment file, feature preproc_descr_file_#{@dataset}"
-    #   exit 1
-    # elsif not(File.readable?(preproc_expname))
-    #   $stderr.puts "Error in the experiment file:"
-    #   $stderr.puts "Parameter preproc_descr_file_#{@dataset} has to be a readable file."
-    #   exit 1
-    # end
-    # preproc_exp = FrPrepConfigData.new(preproc_expname)
-    # @exp.adjoin(preproc_exp)    
-
-    # get the right syntactic interface
-    SynInterfaces.check_interfaces_abort_if_missing(@exp)
-    @interpreter_class = SynInterfaces.get_interpreter_according_to_exp(@exp)
-
-    # initialize grammatical function object (delegating)
-    grf_obj = GrammaticalFunctionAccess.new(@interpreter_class)
-    super(grf_obj)
-
-    # announce the task
-    $stderr.puts "---------"
-    $stderr.puts "Fred experiment #{@exp.get("experiment_ID")}: Featurization of dataset #{@dataset}"
-    if @refeaturize
-      $stderr.puts "Keeping meta-features, redoing featurization only."
-    end
-    if @exp.get("binary_classifiers")
-      $stderr.puts "Writing features for binary classifiers."
-    else
-      $stderr.puts "Writing features for n-ary classifiers."
-    end
-    $stderr.puts "---------"
-
-  end
-
-  ####
-  def compute()
-    if @refeaturize
-      # read meta-feature file,
-      # just redo normal featurization
-      refeaturize()
-    else
-      # write meta features and normal features
-      featurize()
-    end
-  end
-
-  #########################
-  private
-
-  #####
-  # main featurization
-  def featurize()
-
-    ###
-    # make objects
-    unless @exp.get("directory_preprocessed")
-      $stderr.puts "Shalmaneser error: could not find the directory with"
-      $stderr.puts "syntactially preprocessed data."
-      $stderr.puts "Please make sure that 'directory_preprocessed'"
-      $stderr.puts "is set in the frprep experiment file you use with this experiment."
-      exit 1
-    end
-    directory = File.existing_dir(@exp.get("directory_preprocessed"))
-
-    # get context sizes
-    context_sizes = @exp.get_lf("feature", "context")
-    unless context_sizes
-      # no contexts, nothing to compute.
-      # choose default context
-      $stderr.puts "Error: no contexts set."
-      $stderr.puts "I will compute a context of size 1 by default."
-      $stderr.puts "(This goes into the meta-features, but not"
-      $stderr.puts "into the set of features used in the classifier.)"
-      context_sizes = [1]
-    end
-    max_context_size = context_sizes.max()
-
-    # make target determination object
-    if @dataset == "test" and @exp.get("apply_to_all_known_targets")
-      $stderr.puts "Fred: Using all known targets as instances."
-      target_obj = FindAllTargets.new(@exp, @interpreter_class)
-    else
-      if @append_rather_than_overwrite
-        target_obj = FindTargetsFromFrames.new(@exp, @interpreter_class, "a")
-      else
-        target_obj = FindTargetsFromFrames.new(@exp, @interpreter_class, "w")
-      end
-    end
-
-    # make context computation object
-    if @exp.get("single_sent_context")
-      # contexts in the input data doesn't go beyond a single sentence
-      context_obj = SingleSentContextProvider.new(max_context_size, @exp, 
-                                                  @interpreter_class, target_obj, 
-                                                  @dataset)
-      # @todo AB: Put it to the OptionParser, two option are not
-      # compatible, don't do the check here!
-      if @exp.get("noncontiguous_input")
-        $stderr.puts "Warning: 'single_sent_context' has been set in the experiment file."
-        $stderr.puts "So I'm ignoring the 'noncontiguous_input = true' setting."
-      end
-
-    elsif @exp.get("noncontiguous_input")
-      # the input data is not contiguous but 
-      # consists of selected sentences from a larger text
-      context_obj = NoncontiguousContextProvider.new(max_context_size, @exp, 
-                                                     @interpreter_class, target_obj,
-                                                     @dataset)     
-     else
-      # the input data is contiguous, and we're computing contexts not restricted to single sentences
-      context_obj = ContextProvider.new(max_context_size, @exp, 
-                                        @interpreter_class, target_obj, @dataset)
-    end
-
-    zipped_input_dir = fred_dirname(@exp, @dataset, "input_data", "new")
-    
-    ##
-    # make writer object(s)
-
-    writer_classes = [ 
-      MetaFeatureAccess,
-      FredFeatureAccess 
-    ] 
-
-    if @append_rather_than_overwrite
-      # append
-      mode = "a"
-      $stderr.puts "Appending new features to the old"
-
-    else
-      # write
-      mode = "w"
-
-      $stderr.puts "Removing old features for the same experiment (if any)"
-
-      writer_classes.each { |w_class| 
-        w_class.remove_feature_files(@exp, @dataset) 
-      }
-
-      Dir[zipped_input_dir + "*gz"].each { |filename|
-        File.delete(filename)
-      }
-    end
-
-    writers = writer_classes.map { |w_class|
-      w_class.new(@exp, @dataset, mode)
-    }
-
-    ###
-    # zip and store input files
-    Dir[directory + "*.xml"].sort.each { |filename|
-      %x{gzip -c #{filename} > #{zipped_input_dir}#{File.basename(filename)}.gz}
-    }
-
-    # always remember current sentence
-    @current_sent = nil
-    ###
-    # featurize
-
-    # context_obj.each_window yields tuples of:
-    # - a context, an array of tuples [word,lemma, pos, ne]
-    #   string/nil*string/nil*string/nil*string/nil
-    # - ID of main target: string
-    # - target_IDs: array:string, list of IDs of target words
-    # - senses: array:string, the senses for the target
-    # - sent: SalsaTigerSentence object
-    #
-    # for each max. context returned by context object:
-    # determine meta-features:
-    # - context words for all context sizes listed in context_sizes,
-    # - children of target
-    # - parent of target
-    # - siblings of target
-    #
-    # and pass on to writing object(s)
-    target_count = 0
-    context_obj.each_window(directory) { |context, main_target_id, target_ids, senses, sent|
-      # inform user
-      if target_count % 500 == 0
-        $stderr.puts "#{target_count}..."
-      end
-      target_count += 1
-      # determine features
-      feature_hash = Hash.new()
-      compute_target_features(context, max_context_size, feature_hash)
-      compute_context_features(context, max_context_size, context_sizes, feature_hash)
-      compute_syn_features(main_target_id, sent, feature_hash)
-      # write
-      each_lemma_pos_and_senses(senses) { |target_lemma, target_pos, target_sid, target_senses|
-
-        writers.each { |writer_obj|
-
-          writer_obj.write_item(target_lemma, 
-                                target_pos,
-                                target_ids,
-                                target_sid, 
-                                target_senses,
-                                feature_hash)
-        }
-      }
-    }
-    # finalize writers
-    writers.each { |writer_obj|
-      writer_obj.flush()
-    }
-    
-    # record the targets that have been read
-    target_obj.done_reading_targets()
-
-  end
-
-  #####
-  # reuse of meta-features, recompute CSV features
-  def refeaturize()
-
-    ##
-    # remove old features:
-    # normal features only. Keep meta-features.
-    # Don't do anything about zipped input.
-    # Assume it stays as is.
-    if @append_rather_than_overwrite
-      # append
-      mode = "a"
-      $stderr.puts "Appending new features to the old"
-
-    else
-      # write
-      mode = "w"
-
-      $stderr.puts "Removing old features for the same experiment (if any)"
-
-      FredFeatureAccess.remove_feature_files(@exp, @dataset) 
-    end
-
-    ##
-    # read meta-feature file,
-    # write fred feature files
-    meta_reader = MetaFeatureAccess.new(@exp, @dataset, "r")
-    feature_writer = FredFeatureAccess.new(@exp, @dataset, mode)
-
-    ##
-    # featurize
-    target_count = 0
-
-    meta_reader.each_item { |target_lemma, target_pos, target_ids, target_sid, target_senses, feature_hash|
-
-      # inform user
-      if target_count % 500 == 0
-        $stderr.puts "#{target_count}..."
-      end
-      target_count += 1
-
-      feature_writer.write_item(target_lemma, 
-                                target_pos,
-                                target_ids,
-                                target_sid, 
-                                target_senses,
-                                feature_hash)
-    }
-    feature_writer.flush()
-  end
-
-
-  ####
-  # given a list of sense hashes, format
-  # "lex" -> lemma
-  # "pos" -> part of speech
-  # "sense" -> sense
-  #
-  # yield as triples [lemma, pos, sense]
-  def each_lemma_pos_and_senses(shashes)
-
-    # determine target and POS. If we actually have more than one lemma and POS, we're in trouble
-    target_lemmas = shashes.map { |sense_hash| sense_hash["lex"].to_s().gsub(/\s/, "_") }.uniq()
-    target_pos_s =  shashes.map { |sense_hash| sense_hash["pos"].to_s().gsub(/\s/, "_")}.uniq()
-    target_sid =    shashes.map { |sense_hash| sense_hash["sid"].to_s().gsub(/\s/, "_")}.uniq()
-
-    if target_lemmas.length() == 1 and target_pos_s.length() == 1 and target_sid.length() == 1
-
-      yield [target_lemmas.first(), target_pos_s.first(),
-             target_sid.first(),
-             shashes.map { |sense_hash| sense_hash["sense"].to_s().gsub(/\s/, "_") }
-            ]
-
-    else
-      # trouble
-        
-      # group senses by SID, lemma and pos
-      lemmapos2sense = Hash.new
-      shashes.each { |sense_hash|
-        target_lemma = sense_hash["lex"].to_s().gsub(/\s/, "_")
-        target_pos = sense_hash["pos"].to_s().gsub(/\s/, "_")
-        target_sid = sense_hash["sid"].to_s().gsub(/\s/, "_")
-        target_sense = sense_hash["sense"].to_s().gsub(/\s/, "_")
-        key = [target_sid, target_lemma, target_pos]
-        unless lemmapos2sense[key]
-          lemmapos2sense[key] = Array.new()
-        end
-        lemmapos2sense[key] << target_sense
-      }
-
-      # and yield
-      lemmapos2sense.each_key { |target_sid, target_lemma, target_pos|
-        yield [target_lemma, target_pos, target_sid,
-               lemmapos2sense[[target_sid, target_lemma, target_pos]]
-              ]
-      }
-    end
-  end
-  
-  ###
-  # given a context, locate the target,
-  # which is right in the middle,
-  # and enter it into the feature hash
-  #
-  # feature type: TA
-  # entry: word#lemma#pos#ne
-  def compute_target_features(context,      # array: word*lemma*pos*ne
-                              center_pos, # integer: size of context, onesided
-                              feature_hash) # hash: feature_type -> array:feature, enter features here
-  feature_hash["TA"] = [ 
-                          context[center_pos].map { |e| e.to_s() }.join("#").gsub(/\s/, "_")
-                         ]
-  end
-  
-  ###
-  # compute context features:
-  # for each context in the given list of context sizes,
-  # compute a context with feature_type "CXNN" (where NN is the size of the context)
-  # and with features
-  # word#lemma#pos#ne
-  #
-  # enter each context into the feature hash
-  def compute_context_features(context,  # array: word*lemma*pos*ne
-                               center_pos, # int: context is 2*cx_size_onesided + 1 long
-                               context_sizes,    # array:int, produce a context of each of these sizes
-                               feature_hash)     # hash: feature_type -> array:feature, enter features here
-    
-    
-    context_sizes.each { |context_size|
-      # feature type: CXNN, where NN is the size of the context
-      feature_type = "CX" + context_size.to_s()
-      
-      # features: an array of strings
-      feature_hash[feature_type]  = Array.new()
-      
-      # pre-context
-      (center_pos - context_size).upto(center_pos - 1) { |ix|
-        if context[ix]
-          # context entries may be nil at the beginning and end of the text
-          feature_hash[feature_type] << context[ix].map { |e| e.to_s() }.join("#").gsub(/\s/, "_")
-        end
-      }
-      # post-context
-      (center_pos + 1).upto(center_pos + context_size) { |ix|
-        if context[ix]
-          # context entries may be nil at the beginning and end of the text
-          feature_hash[feature_type] << context[ix].map { |e| e.to_s() }.join("#").gsub(/\s/, "_")
-        end
-      }
-    }
-  end
-  
-  ###
-  # compute syntax-dependent features:
-  # children (that is, dependents) of the target word,
-  # parent,
-  # and siblings.
-  #
-  # format:
-  #  feature type is CH for children, PA for parent, SI for siblings
-  #
-  #  individual features are: 
-  #    <dependency>#<word>#<lemma>#<pos>#<ne>
-  def compute_syn_features(main_target_id,  # string: ID of the terminal node that is the target
-                           sent,       # SalsaTigerRegXML object
-                           feature_hash) # hash: feature_type -> array:feature, enter features here
-    
-    target = sent.terminals().detect { |t| t.id() == main_target_id }
-    unless target
-      $stderr.puts "Featurization error: cannot find target with ID #{main_target_id}, skipping."
-      return
-    end
-    
-    # if we're starting a new sentence,
-    # compute dependencies using delegate object for grammatical functions.
-    # also, get_children, get_parents below are methods of the delegate
-    unless sent == @current_sent
-      @current_sent = sent
-      set_sent(sent)
-    end
-    # children
-    feature_hash["CH"] = get_children(target).map { |rel, node|
-      #print "\t", rel, " ", node, "\n"
-      rel.to_s() + "#" + 
-      word_lemma_pos_ne(node, @interpreter_class).map { |e| e.to_s() }.join("#").gsub(/\s/, "_")
-    }
-    
-    # parent
-    feature_hash["PA"] = get_parents(target).map { |rel, node|
-      #print "\t", rel, " ", node, "\n"
-      rel.to_s() + "#" + 
-      word_lemma_pos_ne(node, @interpreter_class).map { |e| e.to_s() }.join("#").gsub(/\s/, "_")
-    }
-    
-    # siblings
-    feature_hash["SI"] = Array.new()
-    get_parents(target).each { |rel, parent|
-      parent_w, d1, d2, d3 = word_lemma_pos_ne(parent, @interpreter_class)
-    
-      get_children(parent).each { |rel, node|
-      #print "\t", rel, " ", node, "\n"
-
-        if node == target
-          next
-        end
-
-        feature_hash["SI"] << parent_w + "#" + 
-        rel.to_s() + "#" + 
-        word_lemma_pos_ne(node, @interpreter_class).map { |e| e.to_s() }.join("#").gsub(/\s/, "_")
-      }
-    }
-  end
-end

data/lib/fred/FredNumTrainingSenses.rb

@@ -1,27 +0,0 @@
-require "fred/FredFeatures"
-
-def determine_training_senses(lemma, exp, lemmas_and_senses_obj, split_id)
-  if split_id
-    # oh no, we're splitting the dataset into random training and test portions.
-    # this means that we actually have to look into the training part of the data to 
-    # determine the number of training senses
-    senses_hash= Hash.new()
-
-    reader = AnswerKeyAccess.new(exp, "train", lemma, "r", split_id, "train")
-    reader.each  { |lemma, pos, ids, sids, gold_senses, transformed_gold_senses|
-      gold_senses.each { |s| senses_hash[s] = true }
-    }
-    return senses_hash.keys()
-
-  else
-    # we're using separate test data.
-    # so we can just look up the number of training senses 
-    # in the lemmas_and_senses object
-    senses = lemmas_and_senses_obj.get_senses(lemma)
-    if senses
-      return senses
-    else
-      return []
-    end
-  end
-end