frprep 0.0.1.prealpha

data/lib/common/RosyConventions.rb

@@ -0,0 +1,171 @@
+# RosyConventions
+# KE May 05
+#
+# Conventions to be used throughout the Rosy system
+# for greater consistency
+
+require "common/ruby_class_extensions"
+
+require "common/EnduserMode"
+
+#################################################################
+#################################################################
+###
+# value restriction (to pass on to a view):
+# some column is restricted to be equal/inequal to some value
+class ValueRestriction
+
+  attr_reader :val_is_variable, :table_name_included
+
+  ###
+  # new(): store values
+  def initialize(column, # string: column name
+		 value,  # value this column is to be restricted to 
+		 var_hash = {}) # hash:additional settings. possible entries:
+                 # posneg: string: "=" or "!=": equality or inequality restriction
+                 #         (default: =)
+                 # table_name_included: boolean: is the table name aready included
+                 #         in the column name? default: false
+
+    @column = column
+    @value = value
+
+    @posneg = var_hash["posneg"]
+    if @posneg.nil?
+      # per default, equality restriction
+      @posneg = "="
+    else
+      unless ["=", "!="].include? @posneg
+        raise "posneg should be either '=' or '!='. I got: " + @posneg.to_s
+      end
+    end
+
+    @table_name_included = var_hash["table_name_included"]
+    if @table_name_included.nil?
+      # per default, the table name is not yet included
+      # in the column name
+      @table_name_included = false
+    end
+
+    # per default, value is a value and not another column name
+    @val_is_variable = false
+  end
+
+  ###
+  # get(): returns a triple [column name(string), eq(string), value(object)]
+  def get()
+    return [@column, @posneg, @value]
+  end
+end
+
+###
+# value restrictions saying that variable1 = variable2:
+# here, value is a variable name, and the table names
+# must be already included
+class VarVarRestriction < ValueRestriction
+  def  initialize(column, value, var_hash={})
+    super(column, value, var_hash)
+    @val_is_variable = true
+    @table_name_included = true
+  end
+end
+
+#################################################################
+#################################################################
+# Table and column names to pass on to a view / SQLQuery:
+# which DB table to access, which columns to view?
+#
+# table_obj: DBTable object or DBWrapper object, table to access.
+#            The important thing is that the object must have a table_name attribute.
+# columns: string|array:string, list of column names, or "*" for all columns
+
+SelectTableAndColumns = Struct.new("SelectTableAndColumns", :table_obj, :columns)
+
+#################################################################
+#################################################################
+
+###
+# transforming feature output to a format that classifiers can handle
+def prepare_output_for_classifiers(string)
+  # change punctuation to _PUNCT_
+  # and change empty space to _
+  # because otherwise some classifiers may spit
+  return string.gsub(/[.":';`]/,"_PUNCT_").gsub(/\s/,"_")
+end
+
+#################################################################
+#################################################################
+
+###
+# classifier directory:
+#  either user-given classifier_dir or our own default classifier directory,
+#  then argrec/arglab/onestep, plus the splitID, if there is one
+def classifier_directory_name(exp,     # RosyConfigData object
+                              step,    # argrec, arglab, onestep
+                              splitID) # string or nil
+
+  if exp.get("classifier_dir")
+    base_dir = File.new_dir(exp.get("classifier_dir"))
+  else
+    base_dir = File.new_dir(exp.instantiate("rosy_dir",
+                                           "exp_ID" => exp.get("experiment_ID")))
+  end
+  classif_base_dir = File.new_dir(base_dir, "classif_dir")
+
+  if splitID
+    return File.new_dir(classif_base_dir, step +  "." + splitID.to_s)
+  else
+    return File.new_dir(classif_base_dir, step)
+  end
+end
+
+#################################################################
+#################################################################
+
+###
+# instance ID: sentence ID plus frame ID
+def construct_instance_id(sentence_id, frame_id)
+  return sentence_id.to_s + "---" + frame_id.to_s
+end
+
+def deconstruct_instance_id(instance_id)
+  return instance_id.split("---")
+end
+
+#################################################################
+#################################################################
+
+# default test ID given when the user didn't specify one
+def default_test_ID()
+  return "apply"
+end
+
+
+#################################################################
+#################################################################
+
+###
+# extend Array class by subsumption
+module Subsumed
+  def subsumed_by?(array2)
+    temp = array2.clone()
+    self.each { |el|
+      found = false
+      temp.each_index { |ix|
+        if el == temp[ix]
+          temp.delete_at(ix)
+          found = true
+          break
+        end
+      }
+      unless found
+        return false
+      end
+    }
+    return true
+  end
+end
+
+class Array
+  include Subsumed
+end

data/lib/common/SQLQuery.rb

@@ -0,0 +1,243 @@
+# class SQLQuery
+# KE, SP 27.1.05
+#
+# provides static methods that generate SQL queries as strings
+# that can then be passed on to the database
+
+require "common/ruby_class_extensions"
+
+require "common/RosyConventions"
+
+class SQLQuery
+  
+
+  #####
+  # SQLQuery.insert
+  #
+  # query created: insert a new row into a given database table
+  # the new row is given as a list of pairs [column_name, value]
+  #
+  # returns: string
+  def SQLQuery.insert(table_name, # string: table name
+		      field_value_pairs) # array: string*object [column_name, cell_value]
+
+    # example:
+    # insert into table01 (field01,field02,field03,field04,field05) values 
+    #  (2, 'second', 'another', '1999-10-23', '10:30:00');
+
+    string = "INSERT INTO " + table_name + "("+
+      field_value_pairs.map { |column_name, cell_value|
+        column_name
+      }.join(",") +
+      ") VALUES (" +
+      field_value_pairs.map { |column_name, cell_value|
+      if cell_value.nil?
+	raise "SQL query construction error: Nil value for column " + column_name
+      end
+      SQLQuery.stringify_value(cell_value)
+    }.join(",") + ");"  
+
+    return string
+  end
+
+  #####
+  # SQLQuery.select
+  #
+  # query created: select from given database tables
+  # all column entries that conform to the given description:
+  # - names of the columns to be selected (or the string "*")
+  # - only those column entries where the row matches the given
+  #   row restrictions: [column_name, column_value] => WHERE column_name IS column_value
+  # - optionally, at most N lines => LIMIT N
+  # - If more than one DB table is named, make a join
+  # - Value restrictions: If it doesn't say which DB table to use,
+  #   use the first one listed in table_col_pairs
+  #
+  # Use with only one database table creates queries like e.g. 
+  #   SELECT column1, column2 FROM table WHERE column3=val3 AND column4!=val4
+  #
+  # or:
+  #   SELECT DISTINCT column1, column2 FROM table WHERE column3=val3 AND column4!=val4 LIMIT 10
+  #
+  # Use with 2 SelectTableAndColumns entries creates queries like
+  #  SELECT table1.column1, table1.column2 FROM table1, table2 WHERE table1.column1=val3 AND table1.id=table2.id 
+  #
+  #
+  # returns: string.
+  #    raises an error if no columns at all are selected
+  def SQLQuery.select(table_col_pairs, # Array: SelectTableAndColumns
+		      row_restrictions, # array: ValueRestriction objects
+		      var_hash = {})  # further parameters:
+    # line_limit: integer: select at most N lines. if nil, all lines are chosen
+    # distinct: boolean: return each tuple only once. if nil or false, duplicates are kept
+
+    if table_col_pairs.empty?
+      raise "Zero tables to select from"
+    end
+
+    ## SELECT
+    string = "SELECT "
+
+    if var_hash["distinct"]
+      # unique return values?
+      string << "DISTINCT "
+    end
+
+    ## column names to select: iterate through table/col pairs
+    at_least_one_column_selected = false
+    string << table_col_pairs.map { |tc|
+
+      if tc.columns == "*"
+	# all columns from this table
+	at_least_one_column_selected = true
+	SQLQuery.prepend_tablename(tc.table_obj.table_name, "*")
+
+      elsif tc.columns.class.to_s == "Array" and not(tc.columns.empty?)
+	# at least one column from this table
+	at_least_one_column_selected = true
+
+	tc.columns.map { |c|
+	  if c.nil? or c.empty?
+	    raise "Got nil/empty value within the column name list"
+	  end
+
+	  SQLQuery.prepend_tablename(tc.table_obj.table_name, c)
+	}.join(", " )
+
+      else
+        # no columns from this table
+        nil
+      end
+    }.compact.join(", ")
+    
+
+    if not(at_least_one_column_selected)
+      raise "Empty select: zero columns selected"
+    end
+
+    ## FROM table name(s)
+    string += " FROM " + table_col_pairs.map { |tc| tc.table_obj.table_name }.join(", ")
+
+    ## WHERE row_restrictions
+    unless row_restrictions.nil? or row_restrictions.empty?  
+      string += " WHERE "+row_restrictions.map { |restr_obj| 
+	# get the actual restriction out of its object
+	# form: name(string) eqsymb(string: =, !=) value(object)
+	name, eqsymb, value = restr_obj.get()
+	if value.nil?
+	  raise "SQL query construction error: Nil value for column " + name
+	end
+        unless restr_obj.val_is_variable
+          # value is a value, not a variable name
+          value = SQLQuery.stringify_value(value)
+        end
+        if restr_obj.table_name_included
+          # name already includes table name, if needed
+          name + eqsymb + value
+        else
+          # prepend name of first table in table_col_pairs
+          SQLQuery.prepend_tablename(table_col_pairs.first.table_obj.table_name(), name) + eqsymb + value
+        end
+      }.join(" AND ")
+    end
+
+
+    ## LIMIT at_most_that_many_lines
+    if var_hash["line_limit"]
+      string += " LIMIT " + var_hash["line_limit"].to_s
+    end
+    string += ";"
+
+    return string
+  end
+
+  #####
+  # SQLQuery.update
+  #
+  # query created: overwrite several cells in possibly multiple rows of a 
+  # database table with new values
+  # rows are selected via row restrictions
+  # 
+  # returns: nothing
+
+  # update table01 set field04=19991022, field05=062218 where field01=1;
+
+  def SQLQuery.update(table_name, # string: table name
+		      field_value_pairs, # array: string*Object: column name and value
+		      row_restrictions # array: ValueRestriction objects: column name and value restriction
+		      )
+    string = "UPDATE "+table_name+" SET "+
+      field_value_pairs.map {|field,value| 
+      if value.nil?
+	raise "SQL query construction error: Nil value for column " + field
+      end
+      field+"="+SQLQuery.stringify_value(value)}.join(", ") +
+      " WHERE "+row_restrictions.map {|restr_obj| 
+	# get the actual restriction out of its object
+	# form: name(string) eqsymb(string: =, !=) value(object)
+	name, eqsymb, value = restr_obj.get()
+	if value.nil?
+	  raise "SQL query construction error: Nil value for column " + name
+	end
+      name + eqsymb + SQLQuery.stringify_value(value)
+    }.join(" AND ")
+    string += ";"
+    return string
+  end
+      
+
+  #####
+  # SQLQuery.add_columns
+  #
+  # query created: extend given table by
+  # one or more columns given by their names and formats
+  #
+  # returns: string
+  def SQLQuery.add_columns(table_name,  # string: table name
+			   column_formats) # array: array: string*string [column_name,column_format]
+    
+    string = "ALTER TABLE " + table_name 
+    string << column_formats.map { |column_name, column_format|
+      " ADD COLUMN " + column_name + " " + column_format
+    }.join(", ")
+    
+    string << ";"
+    
+    return string
+  end
+
+  #####
+  # SQLQuery.stringify ensures that value is a properly
+  # escaped SQL string
+  #
+  # returns: string
+  def SQLQuery.stringify_value(value) # object
+    if value.class == String
+      return "'" + value.gsub(/"/,"QQUOT0").gsub(/'/, "QQUOT1").gsub(/`/, "QQUOT2") + "'"
+    else
+      return value.to_s
+    end
+  end
+
+  #####
+  # SQLQuery.unstringify undoes the result of stringify_value
+  # please apply only to strings
+  def SQLQuery.unstringify_value(value) # string
+    value.gsub(/QQUOT0/, '"').gsub(/QQUOT1/, "'").gsub(/QQUOT2/, "`")
+  end
+
+  ####
+  # SQLQuery.prepend_tablename
+  #
+  # auxiliary method for select:
+  # prepend table name to column name
+  # and if the column name does not already include a table name
+  def SQLQuery.prepend_tablename(table_name, 
+                                 column_name)
+    if not(column_name.include?("."))
+      return table_name + "." + column_name
+    else
+      return column_name
+    end
+  end
+end

data/lib/common/STXmlTerminalOrder.rb

@@ -0,0 +1,194 @@
+#########
+# module StringTerminalsInRightOrder
+#
+# returns the yield of a node, or a list of nodes, as a string
+# of " "-separated words
+#
+# Words are put into the right order, left to right,
+# under the assumption that their node IDs reflect that order
+#
+# Terminal nodes are assumed to have IDs ending in a number,
+# numbered from left to right
+#
+# Splitword nodes are assumed to have IDs ending in N_sM
+# for numbers N and M, where N orders terminals left to right
+# and M orders the splitword parts left to right
+#
+# If the yield of the node/the list of nodes contains all splitwords of a terminal,
+# the whole terminal is taken instead
+#
+# methods:
+# 
+# string_for_node  returns the string for the yield of a node
+#     node: a node object
+#
+# string_for_nodes returns the string for the yield of a list of nodes
+#     nodes: a list of node objects
+
+module StringTerminalsInRightOrder
+  def string_for_node(node)
+    string_for_nodes([node])
+  end
+
+  def string_for_nodes(nodes)
+    a = right_level_terminals_for_nodes(nodes)
+    a = sort_terminals_and_splitwords_left_to_right(a)
+    return node_array_to_string(a)
+  end
+
+  #####
+  private
+  
+  # right_level_terminals_for_nodes:
+  # - compute the yield for each element of 'nodes'
+  # - then consider all splitwords in the yield:
+  #   if all splitwords of a terminal are in the yield,
+  #   then use the terminal rather than its splitwords
+  def right_level_terminals_for_nodes(nodes)
+    a = nodes.map { |n| n.yield_nodes()}.flatten
+    b = Array.new
+    a.each { |n|
+      if n.is_splitword? 
+	# see if a contains all parts of this splitword
+	# if so, take into b the splitword's parent, the terminal,
+	# rather than the individual splitwords
+
+	if n.parent.nil?
+	  # splitword without a parent
+	  b << n
+	elsif b.include? n.parent or a.include? n.parent
+	  # did we already include the splitword's parent in b?
+	  # then we're done
+	else
+
+	  # check if all children of n.parent are in 'a'
+	  all_in = true
+	  n.parent.each_child { |nsibling|
+	    unless a.include? nsibling
+	      all_in = false
+	      break
+	    end
+	  }
+	  
+	  if all_in
+	    # yes, all children of n.parent are in 'a'
+	    b << n.parent
+	  else
+	    # no, some sibling of n is not in 'a'
+	    b << n
+	  end
+	end
+      elsif n.is_terminal?
+	# n is a terminal
+	b << n
+	# if n is anything but a splitword or a terminal,
+	# ignore it
+      end
+    }
+    return b.uniq
+  end
+
+  # sort_terminals_and_splitwords_left_to_right:
+  # take an array of nodes that consists of terminals and splitwords
+  # and sort them using the following comparison:
+  # - when comparing two terminals, use the 
+  #   last numbers in their respective IDs
+  # - when comparing two splitwords, their IDs end in _N_sM
+  #   for numbers N and M.
+  #   If they coincide in N, compare them by M, 
+  #   else compare them by M
+  # - when comparing a terminal and a splitword,
+  #   compare the terminal's last number to the splitword's N
+  def sort_terminals_and_splitwords_left_to_right(nodes)
+    nodes.sort { |a, b|
+      if a.is_splitword? and b.is_splitword?
+	compare_splitwords(a, b)
+      elsif a.is_terminal? and b.is_terminal?
+	compare_terminals(a, b)
+      else
+	compare_mixed(a, b)
+      end
+    }
+  end
+
+  # node_array_to_string:
+  # 'nodes' is an array of node objects, each of which offer a "word" method
+  # string their words together separated by " "
+  def node_array_to_string(nodes)
+    s = ""
+    nodes.each { |n|
+      s = s + n.word + " "
+    }
+    return s
+  end
+
+  # - when comparing two terminals, use the 
+  #   last numbers in their respective IDs
+  def compare_terminals(a, b)
+    last_i(a) <=> last_i(b)
+  end
+
+  # - when comparing two splitwords, their IDs end in _N_sM
+  #   for numbers N and M.
+  #   If they coincide in N, compare them by M, 
+  #   else compare them by M
+  def compare_splitwords(a, b)
+    if splitword_terminal_i(a) == splitword_terminal_i(b)
+      # parts of same terminal?
+      # compare parts
+      last_i(a) <=> last_i(b)
+    else
+      # not parts of same terminal?
+      # compare terminals
+      splitword_terminal_i(a) <=> splitword_terminal_i(b)
+    end
+  end
+
+  # - when comparing a terminal and a splitword,
+  #   compare the terminal's last number to the splitword's N
+  def compare_mixed(a, b)
+    if a.is_splitword? and b.is_terminal?
+      splitword_terminal_i(a) <=> last_i(b)
+
+    elsif a.is_terminal? and b.is_splitword? 
+       last_i(a) <=> splitword_terminal_i(b)
+    else
+      # not one terminal, one splitword?
+      # then what?
+      $stderr.print "SalsaTigerSentence, compare_mixed: confused by "
+      $stderr.print a.id, ",  ", b.id, "\n"
+    end
+  end
+
+  # return last number of the ID of a node
+  def last_i(n)
+    n.id =~ /(\d+)$/ # match final string of digits
+    if $1.nil? # if shouldn't happen _in principle_
+               # but we might get weird node IDs for splitwords; 
+               # so we act gracefully and catch the case where there
+               # is one final letter behind the digits
+      n.id =~ /(\d+)\w$/
+    end
+    if $1.nil? # this shouldn't ever happen
+      $stderr.print "SalsaTigerSentence, last_i: Couldn't extract digits from: "
+      $stderr.print n.id, "\n"
+      exit 1
+    end
+    return $1.to_i       # and return it as number
+  end
+
+  # assume the ID of the node includes N_sM
+  # return N
+  def splitword_terminal_i(n)
+    n.id =~ /(\d+)_s\d*/ # match string of digits before splitword ID
+    if $1.nil? # this shouldn't ever happen
+      $stderr.print "SalsaTigerSentence, splitword_terminal_i: Couldn't extract digits from: "
+      $stderr.print n.id, "\n"
+      exit 1
+    end
+    return $1.to_i       # and return it as number
+  end
+    
+end
+
+