frprep 0.0.1.prealpha

data/lib/rosy/DBWrapper.rb

@@ -0,0 +1,176 @@
+###########################
+# DBWrapper:
+# abstract class wrapping database interfaces,
+# so we can have both an interface to an SQL server
+# and an interface to SQLite in Shalmaneser
+class DBWrapper
+  attr_reader :table_name
+
+  ###
+  def initialize(exp)  # RosyConfigData experiment file object
+    # remember experiment file
+    @exp = exp
+
+    # open the database:
+    # please set to some other value in subclass initialization
+    @database = nil
+
+    # name of default table to access: none
+    @table_name = nil
+  end
+
+  ###
+  # close DB access
+  def close()
+    @database.close()
+  end
+
+  ####
+  # querying the database:
+  # returns an DBResult object
+  def query(query)
+    raise "Overwrite me"
+  end
+
+  ####
+  # querying the database:
+  # no result value
+  def query_noretv(query)
+    raise "Overwrite me"
+  end
+
+  ###
+  # list all tables in the database:
+  # no default here
+  #
+  # returns: list of strings
+  def list_tables()
+    raise "Overwrite me"
+  end
+
+  ###
+  # make a table
+  #
+  # returns: nothing
+  def create_table(table_name, # string
+                   column_formats, # array: array: string*string [column_name,column_format]
+                   index_column_names, # array: string: column_name
+                   indexname)  # string: name of automatically created index column    
+    raise "overwrite me"
+  end
+
+  ###
+  # remove a table
+  def drop_table(table_name)
+    query_noretv("DROP TABLE " + table_name)
+  end
+
+  ###
+  # list all column names of a table (no default)
+  #
+  # returns: array of strings
+  def list_column_names(table_name)
+    return list_column_formats(table_name).map { |col_name, col_format| col_name }
+  end
+
+  #####
+  # list_column_formats
+  #
+  # list column names and column types of this table
+  #
+  # returns: array:string*string, list of pairs [column name, column format]
+  def list_column_formats(table_name)
+    raise "Overwrite me"
+  end
+
+  ####
+  # num_rows
+  #
+  # determine the number of rows in a table
+  # returns: integer
+  def num_rows(table_name)
+    raise "Overwrite me"
+  end
+
+  ####
+  # make a temporary table: basically just make a table
+  #
+  # returns: DBWrapper object (or object of current subclass)
+  # that has the @table_name attribute set to the name of a temporary DB
+  def make_temp_table(column_formats, # array: string*string [column_name,column_format]
+                      index_column_names, # array: string: column_name
+                      indexname)  # string: name of autoincrement primary index
+
+    temp_obj = self.clone()
+    temp_obj.initialize_temp_table(column_formats, index_column_names, indexname)
+    return temp_obj
+  end
+
+  def drop_temp_table()
+    unless @table_name
+      raise "can only do drop_temp_table() for objects that have a temp table"
+    end
+    drop_table(@table_name)
+  end
+
+  ##############################
+  protected
+
+  def initialize_temp_table(column_formats, index_column_names, indexname)
+    @table_name = "t" + Time.new().to_f().to_s().gsub(/\./, "")
+    create_table(@table_name, column_formats, index_column_names, indexname)
+  end
+end
+
+
+
+
+######################################################################
+# DBResult:
+# abstract class keeping query results
+#
+# instantiate for the DB package used
+class DBResult
+  ###
+  # initialize with query result, and keep it
+  def initialize(value)
+    @result = value
+  end
+
+  # column names: NO DEFAULT
+  def list_column_names()
+    raise "Overwrite me"
+  end
+
+  # number of rows: returns an integer
+  def num_rows()
+    return @result.num_rows
+  end
+
+  # yields each row as an array of values
+  def each()
+    @result.each { |row| yield row }
+  end
+
+  # yields each row as a hash: column name=> column value
+  def each_hash()
+    @result.each_hash { |row_hash| yield row_hash }
+  end
+
+  # reset object, such that each() can be run again
+  # DEFAULT DOES NOTHING, PLEASE OVERWRITE
+  def reset()
+  end
+
+  # free result object
+  def free()
+    @result.free()
+  end
+
+  # returns row as an array of column contents
+  def fetch_row()
+    return @result.fetch_row()
+  end
+
+end
+

data/lib/rosy/ExternalConfigData.rb

@@ -0,0 +1,58 @@
+# ExternalConfigData
+# Katrin Erk January 2006
+#
+# All scripts that compute additional external knowledge sources
+# for Fred and Rosy:
+# access to configuration and experiment description file
+
+require 'common/ConfigData'
+
+##############################
+# Class ExternalConfigData
+#
+# inherits from ConfigData,
+# sets variable names appropriate to tasks of external knowledge sources
+
+class ExternalConfigData < ConfigData
+  def initialize(filename)
+
+    # initialize config data object
+    super(filename,          # config file
+	  { "directory" => "string", # features
+
+	    "experiment_id" => "string",
+
+	    "gfmap_restrict_to_downpath" => "bool",
+	    "gfmap_restrict_pathlen" => "integer",
+	    "gfmap_remove_gf" => "list"
+	  },
+	  [] # variables
+	  )
+
+    # set access functions for list features
+    set_list_feature_access("gfmap_remove_gf", 
+			    method("access_as_stringlist"))
+  end
+
+  ###
+  protected
+
+  #####
+  # access_as_stringlist
+  #
+  # assumed format:
+  #
+  #   lhs = rhs1 rhs2 ... rhsN
+  #
+  # given in val_list as string tuples [rhs1,...,rhsN]
+  #
+  # join the rhs strings by spaces, return as string
+  # "rhs1 rhs2 ... rhsN"
+  #
+  def access_as_stringlist(val_list) # array:array:string
+    return val_list.map { |rhs| rhs.join(" ") }
+  end
+end
+
+
+ 

data/lib/rosy/FailedParses.rb

@@ -0,0 +1,130 @@
+# Failed Parses
+#
+# SP May 05
+#
+# Administration of information about failed parses; 
+# - sentence ID
+# - frame
+# - missed FE markables
+#
+# this class is pretty much a gloriefied hash table with methods to 
+# - read FailedParses from a file and to write them to a file
+# - access info in a frame-specific way
+
+class FailedParses
+  
+  ###
+  # initialize
+  #
+  # nothing much happens here
+  def initialize()
+    @failed_parses = Array.new
+  end
+
+  ###
+  # register
+  #
+  # register new failed parse by specifying
+  # - its sentence id (any object)
+  # - its frame (String)
+  # - its FE list (String Array)
+  
+  def register(sent_id, # object
+               frame,   # string: frame name
+               target,  # string?
+               target_pos, # string: target POS
+               fe_list) # array:string
+    if @failed_parses.assoc sent_id
+#      $stderr.puts "Error: trying to register sentence id #{sent_id} twice!"
+#      $stderr.puts "Skipping second occurrence."
+    end
+    @failed_parses << [sent_id,frame,target,target_pos,fe_list]
+  end
+
+  ###
+  # make_split
+  #
+  # produce a "split" of the failed parses into a train and a test section
+  # paramer: train_percentage, Integer between 0 and 100
+  #
+  # returns an Array with two FailedParses objects, the first for the
+  # train data, the second for the test data
+
+  def make_split(train_percentage)
+    unless train_percentage.class < Integer and train_percentage >= 0 and train_percentage <= 100
+      raise "Need Integer between 0 and 100 as training percentage."
+    end
+    train_failed = FailedParses.new()
+    test_failed = FailedParses.new()
+    @failed_parses.each {|sent_id,frame,target,target_pos,fe_list|
+      if rand(100) > train_percentage
+        test_failed.register(sent_id,frame,target,target_pos,fe_list)
+      else
+        train_failed.register(sent_id,frame,target,target_pos,fe_list)
+      end
+    }
+    return [train_failed, test_failed]
+  end
+
+  ###
+  # Access information
+  #
+  # failed_sent: number of failed sentences
+  # failed_fes:  Hash that maps FE names [String] onto numbers of failed FEs [Int] 
+  #
+  # optional parameters: frame, target, target_pos : if not specified or nil, marginal 
+  #                      frequencies are counted (sum over all values)
+  
+
+  def failed_sent(frame_spec=nil,target_spec=nil,target_pos_spec=nil)    
+    counter = 0
+    @failed_parses.each {|sent_id,frame,target,target_pos,fe_list|
+      if ((frame_spec.nil? or frame_spec == frame) and 
+	  (target_spec.nil? or target_spec == target) and 
+	  (target_pos_spec.nil? or target_pos_spec == target_pos))
+	counter += 1
+      end
+    }
+    return counter
+  end
+
+  def failed_fes(frame_spec=nil,target_spec=nil,target_pos_spec=nil)
+    fe_hash = Hash.new(0)
+    @failed_parses.each {|sent_id,frame,target,target_pos,fe_list|
+      if ((frame_spec.nil? or frame_spec == frame) and 
+	  (target_spec.nil? or target_spec == target) and 
+	  (target_pos_spec.nil? or target_pos_spec == target))
+	fe_list.each {|fe_label|
+	  fe_hash[fe_label] += 1
+	}
+      end
+    }
+    return fe_hash
+  end
+
+  
+  ###
+  # Marshalling:
+  #
+  # save - save info about failed parses to file
+  # load - load info about failed parses from file
+
+  def save(filename)
+    io_obj = File.new(filename,"w")
+    Marshal.dump(@failed_parses,io_obj)
+    io_obj.close
+  end
+
+  def load(filename)
+    begin
+      io_obj = File.new(filename)
+      @failed_parses = Marshal.load(io_obj)
+      io_obj.close
+    rescue
+      $stderr.puts "WARNING: couldn't read failed parses file #{filename}."
+      $stderr.puts "I'll assume that there are no failed parses."
+    end
+  end
+ 
+   
+end

data/lib/rosy/FeatureInfo.rb

@@ -0,0 +1,242 @@
+require 'common/ruby_class_extensions'
+
+class RosyFeatureInfo
+  ###
+  # class variable:
+  # list of all known extractors
+  # add to it using add_feature()
+  @@extractors = Array.new
+
+  # boolean. set to true after warning messages have been given once
+  @@warned = false
+
+  ###
+  # add interface/interpreter
+  def RosyFeatureInfo.add_feature(class_name) # Class object
+    @@extractors << class_name
+  end
+  
+  ###
+  def initialize(exp)
+    
+    ##
+    # make list of extractors that are
+    # either required by the user
+    # or needed by the system
+    @current_extractors = Array.new
+    @exp = exp
+
+    # user-chosen extractors:
+    # returns array of pairs [feature group designator(string), options(array:string)]
+    exp.get_lf("feature").each { |extractor_name, options|
+      extractor = @@extractors.detect { |e| e.designator() == extractor_name }
+      unless extractor
+        # no extractor found matching the given designator
+        unless @@warned
+          $stderr.puts "Warning: Could not find a feature extractor for #{extractor_name}: skipping."
+        end
+        next
+      end
+
+      # read and check options
+      step = nil
+
+      options.each { |option|
+        case option
+        when "dontuse", "argrec", "arglab", "onestep"
+
+          if step
+            # step has already been set
+            $stderr.puts "ERROR in feature #{extractor_name}: Please set only one of the options dontuse, argrec, arglab, onestep"
+            exit 1
+          end
+
+          step = option
+
+        else
+          unless @@warned
+            $stderr.puts "Warning: Unknown option for feature #{extractor_name}: #{option}. Skipping"
+          end
+        end
+      }
+
+      @current_extractors << {
+        "extractor" => extractor,
+        "step" => step
+      }
+    }
+
+    # extractors needed by the system
+    @@extractors.select { |e|
+      # select admin features and gold feature
+      ["admin", "gold"].include? e.feature_type()
+    }.each { |extractor|
+      
+      # if we have already added that extractor, remove it
+      # and add it with our own options
+      @current_extractors.delete_if { |descr| descr["extractor"].designator() == extractor.designator() }
+      
+      @current_extractors << {
+        "extractor"=> extractor,
+        "step" => "dontuse"
+      }
+    }
+
+    # make sure that all extractors are computable in the current model
+    # (i.e. check dependencies)
+
+    allstep_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil?
+    }.map { |e| e["extractor"].designator() }
+    argrec_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil? or e_hash["step"] == "argrec"
+    }.map { |e| e["extractor"].designator() }
+    arglab_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil? or e_hash["step"] == "arglab"
+    }.map { |e| e["extractor"].designator() }
+    onestep_extractors = @current_extractors.find_all {|e_hash| e_hash["step"].nil? or e_hash["step"] == "onestep"
+    }.map { |e| e["extractor"].designator() }
+    
+    @current_extractors.delete_if {|extractor_hash|
+      case extractor_hash["step"]
+      when nil
+        computable = extractor_hash["extractor"].is_computable(allstep_extractors)
+      when "argrec"
+        computable = extractor_hash["extractor"].is_computable(argrec_extractors)
+      when "arglab"
+        computable = extractor_hash["extractor"].is_computable(arglab_extractors)
+      when "onestep"
+        computable = extractor_hash["extractor"].is_computable(onestep_extractors)
+      when "dontuse" 
+	# either an admin feature or a user feature not to be used this time
+        computable = true
+      end
+
+      if computable
+        false # i.e. don't delete
+      else
+        unless @@warned
+          $stderr.puts "Warning: Feature extractor #{extractor_hash["extractor"].designator()} cannot be computed: skipping."
+        end
+        true
+      end
+    }
+
+    # make list of all features as hashes
+    # "feature_name" -> string,
+    # "sql_type" -> string,
+    # "is_index" -> boolean,
+    # "step" -> string: argrec, arglab, onestep, or nil
+    # "type" -> string
+    # "phase" -> string: phase 1 or phase 2
+    @features = Array.new
+    @current_extractors.each { |descr|
+      extractor = descr["extractor"]
+      extractor.feature_names.each { |feature_name|
+        @features << {
+          "feature_name" => feature_name,
+          "sql_type"     => extractor.sql_type(),
+          "is_index"     => extractor.info().include?("index"),
+          "step"         => descr["step"],
+          "type"         => extractor.feature_type(),
+          "phase"        => extractor.phase()
+        }
+      }
+    }
+
+    # do not print warnings again if another RosyFeatureInfo object is made
+    @@warned = true
+  end
+
+  ###
+  # get_column_formats
+  #
+  # returns a list of pairs [feature_name(string), sql_column_format(string)]:
+  # all features to be computed, with their SQL column formats
+  def get_column_formats(phase = nil) # string: phase 1 or phase 2
+    return @features.select { |feature_descr|
+      phase.nil? or 
+        feature_descr["phase"] == phase
+    }.map { |feature_descr|
+      [feature_descr["feature_name"], feature_descr["sql_type"]]
+    }
+  end
+
+  ###
+  # get_column_names
+  #
+  # returns a list of feature names (strings)
+  # all features to be computed
+  def get_column_names(phase = nil)  # string: phase 1 or phase 2
+    return @features.select { |feature_descr|
+      phase.nil? or 
+        feature_descr["phase"] == phase
+    }.map { |feature_descr|
+      feature_descr["feature_name"]
+    }
+  end
+
+  ###
+  # get_index_columns
+  #
+  # returns a list of feature (column) names as Strings
+  # consisting of all features that have been requested as index features
+  # in the experiment file or in the list of @@all_features_we_have above
+  def get_index_columns()
+    return @features.select { |feature_descr|
+      feature_descr["is_index"] 
+    }.map {|feature_descr|
+      feature_descr["feature_name"]
+    }
+  end
+
+  ###
+  # get_model_features
+  #
+  # returns a list of feature (column) names as strings
+  # consisting of all the features to be used for the modeling
+  #
+  # step: argrec, arglab, onestep
+  def get_model_features(step)
+
+    return @features.select { |feature_descr|
+      # features for the current step
+      # feature_descr["step"] is argrec, arglab, onestep, dontuse, or nil
+      # nil matches all steps
+      # 'dontuse' matches no step, so these features will never be returned here
+      feature_descr["step"].nil? or
+        feature_descr["step"] == step
+    }.reject { |feature_descr|
+      # that are not admin features or the gold label
+      ["admin", "gold"].include? feature_descr["type"]
+    }.map { |feature_descr|
+      # use just the names of the features
+      feature_descr["feature_name"]
+    }      
+  end
+
+  ###
+  # get_extractor_objects
+  #
+  # returns two lists of feature extractor objects, 
+  # covering all features of the given phase:
+  # the first list contains RosyFeatureExtractor extractors,
+  # the second list contains the others.
+  def get_extractor_objects(phase, # string: "phase 1" or "phase 2"
+                            interpreter_class) # SynInterpreter class
+    unless ["phase 1", "phase 2"].include? phase
+      raise "Shouldn't be here: " + phase
+    end
+
+    return @current_extractors.select { |descr|
+      # select extractors of the right phase
+      descr["extractor"].phase() == phase
+    }.map { |descr|
+
+      # make objects from extractor classes
+      descr["extractor"].new(@exp, interpreter_class)
+    }.distribute { |extractor_obj|
+      # distribute extractors in two bins: 
+      # first, rosy extractors
+      # second, others
+      extractor_obj.class.info().include? "rosy"
+    }
+  end
+end