shalmaneser 1.2.0.rc3 → 1.2.0.rc4

data/lib/common/config_format_element.rb

@@ -0,0 +1,220 @@
+
+##############################
+# ConfigFormatelement is an auxiliary class
+# of ConfigData.
+# It keeps track of feature patterns with variables in them
+# that can be instantiated.
+# @author Andrei Beliankou
+# 
+class ConfigFormatElement
+
+  # given a pattern and a list of variable names,
+  # analyze the pattern and remember the variable names
+  #
+  def initialize(string, # string: feature name, may include names of variables.
+		         # they are included in <>
+		 variables) # list of variable names that can occur
+    
+    @variables = variables
+
+    # pattern: this is what the 'string' is split into,
+    # an array of elements that are either fixed parts or variables.
+    # fixed part: pair [item:string, "string"]
+    # variable: pair [variable_name:string, "variable"]
+    @pattern = Array.new
+    state = "out"
+    item = ""
+
+    # analyze string,
+    # split into variables and fixed parts
+    string.split(//).each { |char|
+
+      case state
+      when "in"
+	case char
+	when "<"
+	  raise "Duplicate < in " + string
+	when ">" 
+	  unless @variables.include? item
+	    raise "Unknown variable " + item
+	  end
+	  @pattern << [item, "variable"]
+	  item = ""
+	  state = "out"
+	else
+	  item << char
+	  state = "in"
+	end
+
+      when "out"
+	case char
+	when "<"
+	  unless item.empty?
+	    @pattern << [item, "string"]
+	    item = ""
+	  end
+	  state = "in"
+	when ">"
+	  raise "Unexpected > in " + string
+	else
+	  item << char
+	  state = "out"
+	end
+
+      else
+	raise "Shouldn't be here"
+      end
+    }
+
+    # read through the whole of "string"
+    # end state has to be "out"
+    unless state == "out"
+      raise "Unclosed < in " + string
+    end
+
+    # last bit still to be recorded?
+    unless item.empty?
+      @pattern << [item, "string"]
+    end
+
+    # make regexp for matching this pattern
+    @regexp = make_regexp(@pattern)
+  end
+
+  # instantiate: given pairs of variable names and variable values,
+  # instantiate @pattern to a string in which var names are replaced
+  # by their values
+  #
+  # returns: string
+  def instantiate(var_hash) # hash variable name(string) => variable value(string)
+
+    # instantiate the pattern
+    return @pattern.map { |item, string_or_var|
+
+      case string_or_var
+      when "string"
+	item
+
+      when "variable"
+	
+	if var_hash[item].nil?
+	  raise "Missing variable instantiation: " + item
+	end
+	var_hash[item]
+
+      else
+	raise "Shouldn't be here"
+      end
+    }.join    
+  end
+
+  # match()
+  #
+  # given a string, try to match it against the @pattern
+  # while setting the variables given in 'fillers' to 
+  # the values given in that hash.
+  #
+  # returns: if the string matches, a hash variable name => value
+  #   that includes the fillers given as a parameter as well as 
+  #   values for all other variables mentioned in @pattern,
+  #   or false if no match.
+  def match(string,   # a string
+	    fillers = nil) # hash variable name(string) => value(string)
+
+    # have we been given partial info about variables?
+    if fillers
+      match = make_regexp(@pattern, fillers).match(string)
+#      $stderr.print "matching " + make_regexp(@pattern, fillers).source + 
+#	" against " + string + " "
+#      if match.nil?
+#	$stderr.puts "no"
+#      else
+#	$stderr.puts "yes"
+#      end      
+    else
+      match = @regexp.match(string)
+    end
+
+    if match.nil?
+      # no match via the regular expression
+      return false
+    end
+
+    # regular expression matched.
+    # construct return value in hash
+    # retv: variable name(string) => value(string)
+    retv = Hash.new()
+    if fillers
+      # include given fillers in retv hash
+      fillers.each_pair { |name, val| retv[name] = val }
+    end
+
+    # now put values for other variables in @pattern into retv
+    index = 1
+    @pattern.to_a.select { |item, string_or_var|
+      string_or_var == "variable"
+    }.select { |item, string_or_var|
+      fillers.nil? or 
+	fillers[item].nil?
+    }.each { |item, string_or_var|
+      # for all items on the pattern list
+      # that are variables and
+      # haven't been filled by the "fillers" list already:
+      # fill from matches
+
+      if match[index].nil?
+	raise "Match, but not enough matched elements? Strange."
+      end
+
+      if retv[item].nil?
+	retv[item] = match[index]
+      else
+	unless retv[item] == match[index]
+	  return false
+	end
+      end
+      
+      index += 1
+    }
+    
+    return retv
+  end
+
+  # used_variables
+  #
+  # returns: an array of variable names used in @pattern
+  def used_variables()
+    return @pattern.select { |item, string_or_var|
+      string_or_var == "variable"
+    }.map { |item, string_or_var| item}
+  end
+
+  ####################
+  private
+
+  # make_regexp:
+  # make regular expression from a pattern
+  # together with some variable fillers
+  #
+  # returns: Regexp object
+  def make_regexp(pattern,  # array of pairs [string, "string"] or [string, "variable"]
+		  fillers = nil) # hash variable name(string) => value(string)
+    return (Regexp.new "^" + 
+      pattern.map { |item, string_or_var|
+      case string_or_var
+      when "variable"
+	if fillers and
+	    fillers[item]
+	  Regexp.escape(fillers[item])
+	else
+	  "(.+)"
+	end
+      when "string"
+	Regexp.escape(item)
+      else
+	raise "Shouldn't be here"
+      end
+    }.join + "$")
+  end
+
+end

data/lib/common/prep_config_data.rb

@@ -0,0 +1,62 @@
+# FPrepConfigData
+# Katrin Erk July 05
+#
+# Preprocessing for Fred and Rosy: 
+# access to a configuration and experiment description file
+
+require "common/config_data"
+
+##############################
+# Class FrPrepConfigData
+#
+# inherits from ConfigData,
+# sets variable names appropriate to preprocessing task
+
+class FrPrepConfigData < ConfigData
+
+  CONFIG_DEFS = {"prep_experiment_ID" => "string", # experiment identifier
+    "frprep_directory" => "string", # dir for frprep internal data
+    # information about the dataset
+    "language" => "string", # en, de
+    "origin"=> "string",    # FrameNet, Salsa, or nothing
+    "format" => "string",   # Plain, SalsaTab, FNXml, FNCorpusXml, SalsaTigerXML
+    "encoding" => "string", # utf8, iso, hex, or nothing
+        
+    # directories
+    "directory_input" => "string", # dir with input data
+    "directory_preprocessed" => "string", # dir with output Salsa/Tiger XML data
+    "directory_parserout" => "string", # dir with parser output for the parser named below
+    
+    # syntactic processing
+    "pos_tagger" => "string", # name of POS tagger
+    "lemmatizer" => "string", # name of lemmatizer
+    "parser" => "string",     # name of parser
+    "pos_tagger_path" => "string", # path to POS tagger
+    "lemmatizer_path" => "string", # path to lemmatizer
+    "parser_path" => "string",     # path to parser
+    "parser_max_sent_num" => "integer", # max number of sentences per parser input file
+    "parser_max_sent_len" => "integer", # max sentence length the parser handles
+    
+    "do_parse" => "bool",    # use parser?
+    "do_lemmatize" => "bool",# use lemmatizer?
+    "do_postag" => "bool",   # use POS tagger?
+    
+    # output format: if tabformat_output == true,
+    # output in Tab format rather than Salsa/Tiger XML
+    # (this will not work if do_parse == true)
+    "tabformat_output" => "bool",
+    
+    # syntactic repairs, dependent on existing semantic role annotation
+    "fe_syn_repair" => "bool", # map words to constituents for FEs: idealize?
+    "fe_rel_repair" => "bool", # FEs: include non-included relative clauses into FEs
+  }
+
+  def initialize(filename)
+    # @param filename [String] path to a config file
+    # @param CONFIG_DEFS [Hash] a list of configuration definitions
+    super(filename, CONFIG_DEFS, [])
+  end
+end
+
+
+ 

data/lib/{common/DBInterface.rb → db/db_interface.rb}

@@ -20,7 +20,7 @@ def get_db_interface(exp, # experiment file object with 'dbtype' entry
   case exp.get("dbtype")
   when "mysql"
     begin
-      require 'rosy/DBMySQL'
+      require 'db/db_mysql'
     rescue
       $stderr.puts "Error loading DB interface."
       $stderr.puts "Make sure you have the Ruby MySQL package installed."
@@ -30,7 +30,7 @@ def get_db_interface(exp, # experiment file object with 'dbtype' entry
     
   when "sqlite"
     begin
-      require 'rosy/DBSQLite'
+      require 'db/db_sqlite'
     rescue
       $stderr.puts "Error loading DB interface."
       $stderr.puts "Make sure you have the Ruby SQLite package installed."

data/lib/{rosy/DBMySQL.rb → db/db_mysql.rb}

@@ -6,8 +6,7 @@
 
 require 'mysql'
 
-
-require 'rosy/DBWrapper'
+require 'db/db_wrapper'
 
 #################
 class DBMySQLResult < DBResult

data/lib/{rosy/DBSQLite.rb → db/db_sqlite.rb}

@@ -6,7 +6,7 @@
 require 'sqlite3'
 require 'tempfile'
 
-require "DBWrapper"
+require 'db/db_wrapper'
 
 #################
 class DBSQLiteResult < DBResult

data/lib/{rosy/DBTable.rb → db/db_table.rb}

@@ -6,7 +6,7 @@
 # Just creating a table, changing the table, and accessing it.
 #
 
-require "common/SQLQuery"
+require 'db/sql_query'
 require "common/RosyConventions"
 
 class DBTable

data/lib/fred/FredBOWContext.rb

@@ -6,14 +6,15 @@ require "common/SynInterfaces"
 require "common/TabFormat"
 require "common/SalsaTigerRegXML"
 require "common/SalsaTigerXMLHelper"
+require "common/RosyConventions"
 
 require 'fred/md5'
-require "fred/FredConfigData"
+require "fred/fred_config_data"
 require "fred/FredConventions"
 require "fred/FredDetermineTargets"
-require "common/DBInterface"
-require "common/RosyConventions"
-require "common/SQLQuery"
+
+require 'db/db_interface'
+require 'db/sql_query'
 
 ########################################
 # Context Provider classes:
@@ -394,6 +395,7 @@ class NoncontiguousContextProvider < AbstractContextProvider
   # yielding contexts.
   def each_window(dir) # string: directory containing Salsa/Tiger XML data
 
+    # @todo AB: Move this chunk to OptionParser.
     # sanity check: do we know where the larger corpus is?
     unless @exp.get("larger_corpus_dir")
       $stderr.puts "Error: 'noncontiguous_input' has been set in the experiment file"
@@ -436,7 +438,7 @@ class NoncontiguousContextProvider < AbstractContextProvider
       # We will need an FrPrep instance and an options object.
       base_dir_path = File.expand_path(File.dirname(__FILE__) + '/../..')
 
-      # Remove this
+      # @todo AB: Remove this
       FileUtils.cp(tf_exp_frprep.path, '/tmp/frprep.exp')
       # after debugging
 
@@ -479,7 +481,7 @@ class NoncontiguousContextProvider < AbstractContextProvider
     # remove temporary data
     temptable_obj.drop_temp_table()
 
-    # AB: TODO Rewrite this passage using pure Ruby.
+    # @todo AB: TODO Rewrite this passage using pure Ruby.
     %x{rm -rf #{frprep_in}}
     %x{rm -rf #{frprep_out}}
     %x{rm -rf #{frprep_dir}}

data/lib/fred/FredDetermineTargets.rb

@@ -1,6 +1,6 @@
 require "fred/FileZipped"
 
-require "fred/FredConfigData"
+require "fred/fred_config_data"
 require "common/SynInterfaces"
 require "fred/FredConventions"
 

data/lib/fred/FredEval.rb

@@ -18,7 +18,7 @@ require "common/Eval"
 require "common/ruby_class_extensions"
 
 # Fred packages
-require "fred/FredConfigData"
+require "fred/fred_config_data"
 require "fred/FredConventions"
 require "fred/FredFeatures"
 require "fred/FredDetermineTargets"

data/lib/fred/FredFeaturize.rb

@@ -29,10 +29,9 @@ require "common/RegXML"
 require "common/SalsaTigerRegXML"
 require "common/SalsaTigerXMLHelper"
 
-require "fred/FredConfigData"
+require "fred/fred_config_data"
 require "fred/FredConventions"
-require "common/FrPrepConfigData"
-require "common/frprep_helper"
+require "common/prep_helper"
 require "common/SynInterfaces"
 
 require "fred/FredBOWContext"
@@ -169,18 +168,24 @@ class FredFeaturize < DelegateClass(GrammaticalFunctionAccess)
     # prepare experiment file: add preprocessing experiment file data
     @exp = exp_obj
 
-    preproc_expname = @exp.get("preproc_descr_file_" + @dataset)
-    if not(preproc_expname)
-      $stderr.puts "Please set the name of the preprocessing exp. file name"
-      $stderr.puts "in the experiment file, feature preproc_descr_file_#{@dataset}"
-      exit 1
-    elsif not(File.readable?(preproc_expname))
-      $stderr.puts "Error in the experiment file:"
-      $stderr.puts "Parameter preproc_descr_file_#{@dataset} has to be a readable file."
-      exit 1
-    end
-    preproc_exp = FrPrepConfigData.new(preproc_expname)
-    @exp.adjoin(preproc_exp)    
+    # @note AB: The following is desabled because we don't want to use
+    #   the dependence on {PrepConfigData}. We duplicate options:
+    #   <do_postag>, <pos_tagger>, <do_lemmatize>, <lemmatizer>,
+    #   <do_parse>, <parser>, <directory_preprocessed>
+    #   in the experiment file of Fred.
+    #
+    # preproc_expname = @exp.get("preproc_descr_file_" + @dataset)
+    # if not(preproc_expname)
+    #   $stderr.puts "Please set the name of the preprocessing exp. file name"
+    #   $stderr.puts "in the experiment file, feature preproc_descr_file_#{@dataset}"
+    #   exit 1
+    # elsif not(File.readable?(preproc_expname))
+    #   $stderr.puts "Error in the experiment file:"
+    #   $stderr.puts "Parameter preproc_descr_file_#{@dataset} has to be a readable file."
+    #   exit 1
+    # end
+    # preproc_exp = FrPrepConfigData.new(preproc_expname)
+    # @exp.adjoin(preproc_exp)    
 
     # get the right syntactic interface
     SynInterfaces.check_interfaces_abort_if_missing(@exp)
@@ -190,7 +195,6 @@ class FredFeaturize < DelegateClass(GrammaticalFunctionAccess)
     grf_obj = GrammaticalFunctionAccess.new(@interpreter_class)
     super(grf_obj)
 
-
     # announce the task
     $stderr.puts "---------"
     $stderr.puts "Fred experiment #{@exp.get("experiment_ID")}: Featurization of dataset #{@dataset}"
@@ -267,6 +271,8 @@ class FredFeaturize < DelegateClass(GrammaticalFunctionAccess)
       context_obj = SingleSentContextProvider.new(max_context_size, @exp, 
                                                   @interpreter_class, target_obj, 
                                                   @dataset)
+      # @todo AB: Put it to the OptionParser, two option are not
+      # compatible, don't do the check here!
       if @exp.get("noncontiguous_input")
         $stderr.puts "Warning: 'single_sent_context' has been set in the experiment file."
         $stderr.puts "So I'm ignoring the 'noncontiguous_input = true' setting."