shalmaneser 1.2.0.rc4 → 1.2.rc5

data/lib/rosy/AbstractFeatureAndExternal.rb

@@ -1,242 +0,0 @@
-# Katrin Erk November 05
-# 
-# Abstract classes for
-# - Rosy features
-# - Rosy interface for external knowledge sources.
-
-require 'rosy/ExternalConfigData'
-
-####
-# Feature Extractor:
-# computes one or more features for a node (a SynNode object) out of
-#  a SalsaTigerSentence
-class AbstractFeatureExtractor
-  @@sent = nil  # SalsaTigerSentence: sentence of the current instance
-  @@frame = nil # FrameNode: frame of the current instance
-  @@node = nil  # SynNode: constituent that is the current instance
-  @@interpreter_class = nil # SynInterpreter class
-  @@instance_ok = true
-
-  ###
-  # returns a string: the designator for this feature extractor
-  # (an extractor may compute several features, but
-  #  in the experiment file it is chosen by a single designator)
-  def AbstractFeatureExtractor.designator()
-    raise "Overwrite me"
-  end
-
-  ###
-  # returns an array of feature names, the names of the 
-  # features that it can compute.
-  # The number of features that the extractor computes must be fixed.
-  def AbstractFeatureExtractor.feature_names()
-    raise "Overwrite me."
-  end
-
-  ###
-  # returns a string: the data type for the feature
-  # to be passed on to the MySQL database,
-  # e.g. VARCHAR(10), INT
-  def AbstractFeatureExtractor.sql_type()
-    raise "Overwrite me"
-  end
-
-  ###
-  # returns a string: the feature type 
-  # (the same for all features computed by this extractor)
-  # possible values:
-  # - gold: gold label
-  # - admin: administrative feature, do not pass this on to the learner
-  # - syn: feature computed from syntactic characteristics of the instance
-  # - sem: feature involving semantic characteristics of the instance
-  # - sentlevel: this feature is the same for all instances of a sentence
-  def AbstractFeatureExtractor.feature_type()
-    raise "Overwrite me"
-  end
-
-  ###
-  # returns a string: "phase 1" or "phase 2",
-  # depending on whether the feature is computed
-  # directly from the SalsaTigerSentence and the SynNode objects
-  # or whether it is computed from the phase 1 features
-  def AbstractFeatureExtractor.phase()
-    raise "Overwrite me."
-  end
-
-  ###
-  # returns an array of strings, providing information about
-  # the feature extractor
-  def AbstractFeatureExtractor.info()
-    return []
-  end
-
-  ###
-  # set sentence, set node, set other settings: 
-  # this is done prior to
-  # feature computation using compute_feature()
-  # such that computations that stay the same for
-  # several features can be done in advance
-  #
-  # This is just relevant for Phase 1
-  #
-  # returns: false/nil if there was a problem
-  def AbstractFeatureExtractor.set_sentence(sent,  # SalsaTigerSentence object
-                                            frame) # FrameNode object
-    @@sent = sent
-    @@frame = frame
-    
-    return true
-  end
-
-  def AbstractFeatureExtractor.set_node(node) # SynNode of the sentence set in set_sentence
-    @@node = node
-
-    return true
-  end
-
-  ###
-  # set sentence, set node, set general settings: this is done prior to
-  # feature computation using compute_feature_value()
-  # such that computations that stay the same for
-  # several features can be done in advance
-  def AbstractFeatureExtractor.set(var_hash = {})
-    # no settings at this point
-    
-    return true
-  end
-  # test during initialisation whether a feature is computable
-  # gives the feature the possibility to specify additional constraints
-  # e.g. for phase2 features : specify which extractors from phase 1 are presupposed
-  def AbstractFeatureExtractor.is_computable(extractor_list) # bool
-    return true
-  end
-
-  ###
-  # @param exp [ConfigData] Experiment file information
-  # @param interpreter_class [Class]
-  def initialize(exp, interpreter_class)
-    @exp = exp
-    @@interpreter_class = interpreter_class
-  end
-
-  ###
-  # compute: compute features
-  #
-  # returns an array of features (strings), length the same as the
-  # length of feature_names()
-  def compute_features()
-    raise "overwrite me"
-  end
-
-  ###
-  # phase 2 extractors: 
-  # compute features for a complete view
-  #
-  # returns: an array of columns,
-  # where a column is an array of feature values.
-  # returns one column per entry in feature_names()
-  def compute_features_on_view(view) # DBView object
-    raise "overwrite me"
-  end
-
-  # At this place, we had abstract methods for "training" phase 2 features 
-  # Since this involves introducing a "state" that is nontrivial to preserve
-  # for a standalone version of the classifiers, without keeping the training data,
-  # we decided to remove this functionality (30.11.05).
-  # Features which rely on learning patterns from the training data and applying them
-  # to the test data will from now on be implemented as externals.
-
-  ######
-  protected
-
-  def AbstractFeatureExtractor.announce_me()
-    # AB: In 1.9 constants are symbols.
-    if Module.constants.include?("RosyFeatureInfo") or Module.constants.include?(:RosyFeatureInfo)
-      # yup, we have a class to which we can announce ourselves
-      RosyFeatureInfo.add_feature(eval(self.name()))
-    else
-      # no interface collector class
-#      $stderr.puts "Feature #{self.name()} not announced: no RosyFeatureInfo."
-    end
-  end
-end
-
-################################################################
-# Wrapper class for extractors that compute a single feature
-class AbstractSingleFeatureExtractor < AbstractFeatureExtractor
-  
-  ###
-  # returns a string: the designator for this feature extractor
-  # (an extractor may compute several features, but
-  #  in the experiment file it is chosen by a single designator)
-  #
-  # here: single feature, and the feature name is the designator
-  def AbstractFeatureExtractor.designator()
-    return eval(self.name()).feature_name()
-  end
-
-  ###
-  def AbstractSingleFeatureExtractor.feature_names()
-    return [eval(self.name()).feature_name()]
-  end
-
-  ###
-  def compute_features()
-    return [compute_feature()]
-  end
-
-  def compute_features_on_view(view) # DBView object
-    return [compute_feature_on_view(view)]
-  end
-
-
-  ######
-  # Single-feature methods
-
-  ###
-  def AbstractSingleFeatureExtractor.feature_name()
-    raise "Overwrite me."
-  end
-  
-  ###
-  def compute_feature()
-    raise "Overwrite me"
-  end
-
-  ###
-  def compute_feature_on_view(view) # DBView object
-    raise "Overwrite me"
-  end
-end
-
-######################################################
-
-class ExternalFeatureExtractor < AbstractFeatureExtractor
-
-  @@warning_uttered = false
-
-  ####
-  # initialization:
-  #
-  # read experiment file for external interfaces
-  def initialize(exp,    # RosyConfigData object
-                 interpreter_class)
-
-    @exp_rosy = exp
-    @@interpreter_class = interpreter_class
-
-    unless @exp_rosy.get("external_descr_file")
-      unless @@warning_uttered
-	$stderr.puts "Warning: Cannot compute external feature"
-	$stderr.puts "since 'external_descr_file' has not been set"
-	$stderr.puts "in the Rosy experiment file."
-	@@warning_uttered = true
-      end
-
-      @exp_external = nil
-      return
-    end
-
-    @exp_external = ExternalConfigData.new(@exp_rosy.get("external_descr_file"))
-  end
-end

data/lib/rosy/ExternalConfigData.rb

@@ -1,58 +0,0 @@
-# ExternalConfigData
-# Katrin Erk January 2006
-#
-# All scripts that compute additional external knowledge sources
-# for Fred and Rosy:
-# access to configuration and experiment description file
-
-require 'common/config_data'
-
-##############################
-# Class ExternalConfigData
-#
-# inherits from ConfigData,
-# sets variable names appropriate to tasks of external knowledge sources
-
-class ExternalConfigData < ConfigData
-  def initialize(filename)
-
-    # initialize config data object
-    super(filename,          # config file
-	  { "directory" => "string", # features
-
-	    "experiment_id" => "string",
-
-	    "gfmap_restrict_to_downpath" => "bool",
-	    "gfmap_restrict_pathlen" => "integer",
-	    "gfmap_remove_gf" => "list"
-	  },
-	  [] # variables
-	  )
-
-    # set access functions for list features
-    set_list_feature_access("gfmap_remove_gf", 
-			    method("access_as_stringlist"))
-  end
-
-  ###
-  protected
-
-  #####
-  # access_as_stringlist
-  #
-  # assumed format:
-  #
-  #   lhs = rhs1 rhs2 ... rhsN
-  #
-  # given in val_list as string tuples [rhs1,...,rhsN]
-  #
-  # join the rhs strings by spaces, return as string
-  # "rhs1 rhs2 ... rhsN"
-  #
-  def access_as_stringlist(val_list) # array:array:string
-    return val_list.map { |rhs| rhs.join(" ") }
-  end
-end
-
-
- 

data/lib/rosy/FailedParses.rb

@@ -1,130 +0,0 @@
-# Failed Parses
-#
-# SP May 05
-#
-# Administration of information about failed parses; 
-# - sentence ID
-# - frame
-# - missed FE markables
-#
-# this class is pretty much a gloriefied hash table with methods to 
-# - read FailedParses from a file and to write them to a file
-# - access info in a frame-specific way
-
-class FailedParses
-  
-  ###
-  # initialize
-  #
-  # nothing much happens here
-  def initialize()
-    @failed_parses = Array.new
-  end
-
-  ###
-  # register
-  #
-  # register new failed parse by specifying
-  # - its sentence id (any object)
-  # - its frame (String)
-  # - its FE list (String Array)
-  
-  def register(sent_id, # object
-               frame,   # string: frame name
-               target,  # string?
-               target_pos, # string: target POS
-               fe_list) # array:string
-    if @failed_parses.assoc sent_id
-#      $stderr.puts "Error: trying to register sentence id #{sent_id} twice!"
-#      $stderr.puts "Skipping second occurrence."
-    end
-    @failed_parses << [sent_id,frame,target,target_pos,fe_list]
-  end
-
-  ###
-  # make_split
-  #
-  # produce a "split" of the failed parses into a train and a test section
-  # paramer: train_percentage, Integer between 0 and 100
-  #
-  # returns an Array with two FailedParses objects, the first for the
-  # train data, the second for the test data
-
-  def make_split(train_percentage)
-    unless train_percentage.class < Integer and train_percentage >= 0 and train_percentage <= 100
-      raise "Need Integer between 0 and 100 as training percentage."
-    end
-    train_failed = FailedParses.new()
-    test_failed = FailedParses.new()
-    @failed_parses.each {|sent_id,frame,target,target_pos,fe_list|
-      if rand(100) > train_percentage
-        test_failed.register(sent_id,frame,target,target_pos,fe_list)
-      else
-        train_failed.register(sent_id,frame,target,target_pos,fe_list)
-      end
-    }
-    return [train_failed, test_failed]
-  end
-
-  ###
-  # Access information
-  #
-  # failed_sent: number of failed sentences
-  # failed_fes:  Hash that maps FE names [String] onto numbers of failed FEs [Int] 
-  #
-  # optional parameters: frame, target, target_pos : if not specified or nil, marginal 
-  #                      frequencies are counted (sum over all values)
-  
-
-  def failed_sent(frame_spec=nil,target_spec=nil,target_pos_spec=nil)    
-    counter = 0
-    @failed_parses.each {|sent_id,frame,target,target_pos,fe_list|
-      if ((frame_spec.nil? or frame_spec == frame) and 
-	  (target_spec.nil? or target_spec == target) and 
-	  (target_pos_spec.nil? or target_pos_spec == target_pos))
-	counter += 1
-      end
-    }
-    return counter
-  end
-
-  def failed_fes(frame_spec=nil,target_spec=nil,target_pos_spec=nil)
-    fe_hash = Hash.new(0)
-    @failed_parses.each {|sent_id,frame,target,target_pos,fe_list|
-      if ((frame_spec.nil? or frame_spec == frame) and 
-	  (target_spec.nil? or target_spec == target) and 
-	  (target_pos_spec.nil? or target_pos_spec == target))
-	fe_list.each {|fe_label|
-	  fe_hash[fe_label] += 1
-	}
-      end
-    }
-    return fe_hash
-  end
-
-  
-  ###
-  # Marshalling:
-  #
-  # save - save info about failed parses to file
-  # load - load info about failed parses from file
-
-  def save(filename)
-    io_obj = File.new(filename,"w")
-    Marshal.dump(@failed_parses,io_obj)
-    io_obj.close
-  end
-
-  def load(filename)
-    begin
-      io_obj = File.new(filename)
-      @failed_parses = Marshal.load(io_obj)
-      io_obj.close
-    rescue
-      $stderr.puts "WARNING: couldn't read failed parses file #{filename}."
-      $stderr.puts "I'll assume that there are no failed parses."
-    end
-  end
- 
-   
-end

data/lib/rosy/FeatureInfo.rb

@@ -1,242 +0,0 @@
-require 'common/ruby_class_extensions'
-
-class RosyFeatureInfo
-  ###
-  # class variable:
-  # list of all known extractors
-  # add to it using add_feature()
-  @@extractors = Array.new
-
-  # boolean. set to true after warning messages have been given once
-  @@warned = false
-
-  ###
-  # add interface/interpreter
-  def RosyFeatureInfo.add_feature(class_name) # Class object
-    @@extractors << class_name
-  end
-  
-  ###
-  def initialize(exp)
-    
-    ##
-    # make list of extractors that are
-    # either required by the user
-    # or needed by the system
-    @current_extractors = Array.new
-    @exp = exp
-
-    # user-chosen extractors:
-    # returns array of pairs [feature group designator(string), options(array:string)]
-    exp.get_lf("feature").each { |extractor_name, options|
-      extractor = @@extractors.detect { |e| e.designator() == extractor_name }
-      unless extractor
-        # no extractor found matching the given designator
-        unless @@warned
-          $stderr.puts "Warning: Could not find a feature extractor for #{extractor_name}: skipping."
-        end
-        next
-      end
-
-      # read and check options
-      step = nil
-
-      options.each { |option|
-        case option
-        when "dontuse", "argrec", "arglab", "onestep"
-
-          if step
-            # step has already been set
-            $stderr.puts "ERROR in feature #{extractor_name}: Please set only one of the options dontuse, argrec, arglab, onestep"
-            exit 1
-          end
-
-          step = option
-
-        else
-          unless @@warned
-            $stderr.puts "Warning: Unknown option for feature #{extractor_name}: #{option}. Skipping"
-          end
-        end
-      }
-
-      @current_extractors << {
-        "extractor" => extractor,
-        "step" => step
-      }
-    }
-
-    # extractors needed by the system
-    @@extractors.select { |e|
-      # select admin features and gold feature
-      ["admin", "gold"].include? e.feature_type()
-    }.each { |extractor|
-      
-      # if we have already added that extractor, remove it
-      # and add it with our own options
-      @current_extractors.delete_if { |descr| descr["extractor"].designator() == extractor.designator() }
-      
-      @current_extractors << {
-        "extractor"=> extractor,
-        "step" => "dontuse"
-      }
-    }
-
-    # make sure that all extractors are computable in the current model
-    # (i.e. check dependencies)
-
-    allstep_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil?
-    }.map { |e| e["extractor"].designator() }
-    argrec_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil? or e_hash["step"] == "argrec"
-    }.map { |e| e["extractor"].designator() }
-    arglab_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil? or e_hash["step"] == "arglab"
-    }.map { |e| e["extractor"].designator() }
-    onestep_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil? or e_hash["step"] == "onestep"
-    }.map { |e| e["extractor"].designator() }
-    
-    @current_extractors.delete_if {|extractor_hash|
-      case extractor_hash["step"]
-      when nil
-        computable = extractor_hash["extractor"].is_computable(allstep_extractors)
-      when "argrec"
-        computable = extractor_hash["extractor"].is_computable(argrec_extractors)
-      when "arglab"
-        computable = extractor_hash["extractor"].is_computable(arglab_extractors)
-      when "onestep"
-        computable = extractor_hash["extractor"].is_computable(onestep_extractors)
-      when "dontuse" 
-	# either an admin feature or a user feature not to be used this time
-        computable = true
-      end
-
-      if computable
-        false # i.e. don't delete
-      else
-        unless @@warned
-          $stderr.puts "Warning: Feature extractor #{extractor_hash["extractor"].designator()} cannot be computed: skipping."
-        end
-        true
-      end
-    }
-
-    # make list of all features as hashes
-    # "feature_name" -> string,
-    # "sql_type" -> string,
-    # "is_index" -> boolean,
-    # "step" -> string: argrec, arglab, onestep, or nil
-    # "type" -> string
-    # "phase" -> string: phase 1 or phase 2
-    @features = Array.new
-    @current_extractors.each { |descr|
-      extractor = descr["extractor"]
-      extractor.feature_names.each { |feature_name|
-        @features << {
-          "feature_name" => feature_name,
-          "sql_type"     => extractor.sql_type(),
-          "is_index"     => extractor.info().include?("index"),
-          "step"         => descr["step"],
-          "type"         => extractor.feature_type(),
-          "phase"        => extractor.phase()
-        }
-      }
-    }
-
-    # do not print warnings again if another RosyFeatureInfo object is made
-    @@warned = true
-  end
-
-  ###
-  # get_column_formats
-  #
-  # returns a list of pairs [feature_name(string), sql_column_format(string)]:
-  # all features to be computed, with their SQL column formats
-  def get_column_formats(phase = nil) # string: phase 1 or phase 2
-    return @features.select { |feature_descr|
-      phase.nil? or 
-        feature_descr["phase"] == phase
-    }.map { |feature_descr|
-      [feature_descr["feature_name"], feature_descr["sql_type"]]
-    }
-  end
-
-  ###
-  # get_column_names
-  #
-  # returns a list of feature names (strings)
-  # all features to be computed
-  def get_column_names(phase = nil)  # string: phase 1 or phase 2
-    return @features.select { |feature_descr|
-      phase.nil? or 
-        feature_descr["phase"] == phase
-    }.map { |feature_descr|
-      feature_descr["feature_name"]
-    }
-  end
-
-  ###
-  # get_index_columns
-  #
-  # returns a list of feature (column) names as Strings
-  # consisting of all features that have been requested as index features
-  # in the experiment file or in the list of @@all_features_we_have above
-  def get_index_columns()
-    return @features.select { |feature_descr|
-      feature_descr["is_index"] 
-    }.map {|feature_descr|
-      feature_descr["feature_name"]
-    }
-  end
-
-  ###
-  # get_model_features
-  #
-  # returns a list of feature (column) names as strings
-  # consisting of all the features to be used for the modeling
-  #
-  # step: argrec, arglab, onestep
-  def get_model_features(step)
-
-    return @features.select { |feature_descr|
-      # features for the current step
-      # feature_descr["step"] is argrec, arglab, onestep, dontuse, or nil
-      # nil matches all steps
-      # 'dontuse' matches no step, so these features will never be returned here
-      feature_descr["step"].nil? or
-        feature_descr["step"] == step
-    }.reject { |feature_descr|
-      # that are not admin features or the gold label
-      ["admin", "gold"].include? feature_descr["type"]
-    }.map { |feature_descr|
-      # use just the names of the features
-      feature_descr["feature_name"]
-    }      
-  end
-
-  ###
-  # get_extractor_objects
-  #
-  # returns two lists of feature extractor objects, 
-  # covering all features of the given phase:
-  # the first list contains RosyFeatureExtractor extractors,
-  # the second list contains the others.
-  def get_extractor_objects(phase, # string: "phase 1" or "phase 2"
-                            interpreter_class) # SynInterpreter class
-    unless ["phase 1", "phase 2"].include? phase
-      raise "Shouldn't be here: " + phase
-    end
-
-    return @current_extractors.select { |descr|
-      # select extractors of the right phase
-      descr["extractor"].phase() == phase
-    }.map { |descr|
-
-      # make objects from extractor classes
-      descr["extractor"].new(@exp, interpreter_class)
-    }.distribute { |extractor_obj|
-      # distribute extractors in two bins: 
-      # first, rosy extractors
-      # second, others
-      extractor_obj.class.info().include? "rosy"
-    }
-  end
-end