shalmaneser-lib 1.2.rc5

data/lib/db/db_table.rb

@@ -0,0 +1,237 @@
+# class DBTable
+# KE, SP 27.1.05
+#
+# Manages one table in a (given) SQL database
+# Doesn't know anything about the ROSY application
+# Just creating a table, changing the table, and accessing it.
+#
+
+require 'db/sql_query'
+# require "RosyConventions"
+
+class DBTable
+  attr_reader :index_name, :table_name
+
+  #####
+  # new
+  #
+  # creates the table for this object.
+  # The name of the table (given as parameter) can be new, in which caes the table
+  # is created, or old, in which case we check whether its format matches the format
+  # given in the parameters.
+  #
+  # The table format is given in the form of column formats (column names and column formats,
+  # formats are the usual SQLy things). Additionally, a subset of the column names can be
+  # designated index columns, which means that the table is indexed (and can be searched quickly)
+  # for them.
+  #
+  # DBTable internally constructs a "Primary index" feature that is called "XXindexXX" (autoincrement column)
+  #
+  # For all columns that are added later using add_columns, DBTable adds a prefix to the column names;
+  # these columns are not checked against the column_formats when opening an existing table;
+  # this can be used to store experiment-specific data.
+
+  def initialize(db_obj, # DBWrapper object
+                 table_name, # string: name of DB table (existing/new)
+                 mode,       # new: starts new DB table, removes old if it exists. open: reopens existing DB table
+                 hash={})    # hash: parameter name => parameter value, depending on mode
+    # mode= new needs:
+    #  'col_formats': array:array len 2: string*string, [column_name, column_format]
+    #  'index_cols':  array:string: column_names that should be used to index the table
+    #  'addcol_prefix': string: prefix for names of additional columns
+    # mode='open' needs:
+    #  'col_formats': array: string*string: column names/formats
+    #               May be nil, in that case column name match isn't tested
+
+    @index_name = "XXindexXX"
+    @db_obj = db_obj
+    @table_name = table_name
+
+    case mode
+    when 'new'
+      ###
+      # open new database
+
+      # sanity check: exactly the required parameters present?
+      unless hash.keys.sort == ['addcol_prefix', 'col_formats', 'index_cols']
+        raise "Expecting hash parameters 'addcol_prefix', 'col_formats', 'index_cols'.\n" +
+              "I got: " + hash.keys.join(", ")
+      end
+
+      # sanity check: main index column name should be unique
+      all_column_names = hash['col_formats'].map { |name, format| name}
+      if all_column_names.include? @index_name
+        raise "[DBTable] You used the reserved name #{@index_name} as a column name. Please don't do that!"
+      end
+
+      # sanity check: index_column_names should be included in column_names
+      hash['index_cols'].each { |name|
+        unless all_column_names.include? name
+          raise "[DBTable] #{name} is in the list of index names, but it isn't in the list of column names."
+        end
+      }
+
+      # does a table with name table_name exist? if so, remove it
+      if @db_obj.list_tables.include? table_name
+        # this table exists
+        # remove old table
+        @db_obj.drop_table(table_name)
+      end
+
+      @db_obj.create_table(table_name,hash['col_formats'],
+                           hash['index_cols'], @index_name)
+    when 'open'
+
+      ###
+      # open existing database table
+
+      # sanity check: exactly the required parameters present?
+      hash.keys.each { |key|
+        unless ['addcol_prefix', 'col_names'].include? key
+          raise "Expecting hash parameters 'addcol_prefix', 'col_names'.\n" +
+                "I got: " + hash.keys.join(", ")
+        end
+      }
+      # sanity check: main index column name should be unique
+      if hash['col_names'] && hash['col_names'].include?(@index_name)
+        raise "[DBTable] You used the reserved name #{@index_name} as a column name. Please don't do that!"
+      end
+
+      # does a table with name table_name exist?
+      unless @db_obj.list_tables.include? table_name
+        raise "[DBTable] Sorry, I cannot find a database table named #{table_name}."
+      end
+
+      # check if all column formats match
+
+      if hash['col_names']
+
+        existing_fields = @db_obj.list_column_names(table_name).reject { |col|
+          col =~ /^#{hash["addcol_prefix"]}/ or
+            col == @index_name
+        }
+
+        unless existing_fields.sort == hash["col_names"].sort
+          raise "[DBTable] Column names in the DB table #{table_name}\n" +
+                "don't match feature specification in the experiment file.\n" +
+                "Table:\n\t" + existing_fields.sort.join(", ") +
+                "\n\nExp. file:\n\t" + hash["col_names"].sort.join(", ")
+        end
+
+      else
+        # no column names given, no check of column formats
+      end
+
+    else
+      raise "Parameter 'mode' needs to be either 'new' or 'open'! I got " + mode.to_s
+    end
+  end
+
+  #####
+  # list_column_names
+  #
+  # list column names of this table
+  #
+  # returns: array:string, list of column names
+  def list_column_names
+    return @db_obj.list_column_names(@table_name)
+  end
+
+  #####
+  # list_column_formats
+  #
+  # list column names and column types of this table
+  #
+  # returns: array:string*string, list of pairs [column name, column format]
+  def list_column_formats
+    return @db_obj.list_column_formats(@table_name)
+  end
+
+  #####
+  # change_format_add_columns
+  #
+  # adds one or more columns to the table managed by this object
+  # columns are given by their names and formats, as above
+  #
+  # returns: nothing
+  def change_format_add_columns(column_formats) # array: string*string [column_name,column_format]
+
+    if column_formats.nil? or column_formats.empty?
+      raise "Need nonempty column_formats list"
+    end
+
+    column_formats.each { |col_name, col_format|
+      unless col_name =~ /^#{@addcol_prefix}/
+        raise "Columns that are added need to have prefix #{@addcol_prefix}!"
+      end
+    }
+
+    execute_command(SQLQuery.add_columns(@table_name, column_formats))
+  end
+
+  #####
+  # change_format_remove_column
+  #
+  # removes one column from the table managed by this object
+  #
+  # returns: nothing
+  def change_format_remove_column(column_name) # string:name of the column to remove
+    unless list_column_names(@table_name).include? column_name
+      $stderr.puts "WARNING: Cannot remove column #{column_name}: I don't have it"
+      return
+    end
+
+    execute_command("ALTER TABLE #{@table_name} DROP COLUMN #{column_name}")
+  end
+
+
+  #####
+  # insert_row
+  #
+  # inserts a new row into the table and fills cells with values, as specified
+  # by the column_value_pairs
+  #
+  # returns: nothing
+  def insert_row(column_value_pairs) # array: string*Object [column_name,column_value]
+    if column_value_pairs.nil? or column_value_pairs.empty?
+      raise "Need nonempty column_value_pairs list"
+    end
+    execute_command(SQLQuery.insert(@table_name,column_value_pairs))
+  end
+
+  #####
+  # update_row
+  #
+  # update column values for a given row which is identified
+  # via its (autoincrement) index
+  #
+  # returns: nothing
+  def update_row(index, # index, content of autoincrement column
+                 column_value_pairs) # array: string*Object [column_name, column_value]
+
+    if column_value_pairs.nil? or column_value_pairs.empty?
+      raise "Need nonempty column_value_pairs list"
+    end
+    execute_command(SQLQuery.update(@table_name,
+                                    column_value_pairs,
+                                    [ValueRestriction.new(@index_name, index)]))
+  end
+
+  ####
+  private
+
+  ###
+  # execute_command:
+  # execute DB command
+  #
+  # returns nil: the commands in this package are all
+  # not of the kind that requires a return value
+  def execute_command(command)
+    begin
+      @db_obj.query_noretv(command)
+    rescue
+      $stderr.puts "Error executing SQL query. Command was:\n" + command
+      exit 1
+    end
+  end
+end

data/lib/db/db_view.rb

@@ -0,0 +1,416 @@
+# class DBView
+# KE, SP 27.1.05
+#
+# builds on class DBTable, which offers access to a database table
+# extract views of the table (select columns, select rows)
+# and offers access methods for these views.
+# Rows of the table can be returned either as hashes or as arrays.
+#
+# There is a special column of the table (the name of which we get in the new() method),
+# the gold column.
+# It can be returned directly, or modified by some "dynamic feature object",
+# and its value (modified or unmodified) will always be last in the array representation of a row.
+
+require 'db/sql_query'
+require "ruby_class_extensions"
+# require "RosyConventions"
+require 'db/select_table_and_columns'
+
+class DBView
+
+  ################
+  # new
+  #
+  # prepare a view.
+  # given a list of DB tables to access, each with its
+  #   set of features to be returned in the view,
+  # a set of value restrictions,
+  # the name of the gold feature,
+  # and a list of objects that manipulate the gold feature into alternate
+  # gold features.
+  #
+  # value_restrictions restricts the view to those rows for which the value restrictions hold,
+  # e.g. only those rows where frame = Bla, or only those rows where partofspeech = Blupp
+  #
+  # The view remembers the indices of the _first_ table in the list of tables
+  # it is given.
+  #
+  # A standard dynamic ID can be given: DynGold objects all have an id() method,
+  # which returns a string, by which the use of the object can be requested
+  # of the view. If no dynamic ID is given below in methods each_array,
+  # each_hash, each_sentence, the system falls back to the standard dynamic ID.
+  # if none is given here, the standard DynGold object is the one that doesn't
+  # change the gold column. If one is given here, it will be used by default
+  # when no ID is given in each_hash, each_array, each_sentence
+  #
+  # The last parameter is a hash with the following optional entries:
+  #  "gold":
+  #     string: name of the gold feature
+  #     If you want the gold feature to be mapped using a DynGold object,
+  #     you need to specify this parameter -- and you need to include
+  #     the gold feature in some feature_list.
+  #     Warning: if a feature of this name appears in several of the
+  #     feature lists, only the first one is mapped
+  #  "dynamic_feature_list":
+  #     array:DynGold objects, list of objects that map the gold feature
+  #     to a different feature value (e.g. to "FE", "NONE")
+  #     DynGold objects have one method make: string -> string
+  #     that maps one gold feature,
+  #     and one method id: -> string that gives an ID unique to this DynGold class
+  #     and by which this DynGold class can be chosen.
+  # "standard_dyngold_id":
+  #     string: standard DynGold object ID (see above)
+  # "sentence_id_feature":
+  #      string: feature name for the sentence ID column, needed for each_sentence()
+  #
+  # further parameters that are passed on to SQLQuery.select: see there
+
+  def initialize(table_col_pairs, # array:SelectTableAndColumns objects
+                 value_restrictions, # array:ValueRestriction objects
+                 db_obj, # MySql object (from mysql.rb) that already has access to the correct database
+                 parameters = {}) # hash with further parameters: see above
+
+    @db_obj = db_obj
+    @table_col_pairs = table_col_pairs
+    @parameters = parameters
+
+    # view empty?
+    if @table_col_pairs.empty? or
+        @table_col_pairs.big_and { |tc| tc.columns.is_a?(Array) and tc.columns.empty? }
+      @view_empty = true
+      return
+    else
+      @view_empty = false
+    end
+
+    # okay, we can make the view, it contains at least one table and
+    # at least one column:
+    # do one view for all columns requested, and one for the indices of each table
+    #
+    # @main_table is a DBResult object
+    @main_table = execute_command(SQLQuery.select(@table_col_pairs,
+                                                  value_restrictions, parameters))
+
+    # index_tables: Hash: table name =>  DBResult object
+    @index_tables = {}
+    table_col_pairs.each_with_index { |tc, index|
+      # read index column of this table, add all the other tables
+      # with empty column lists
+      index_table_col_pairs = @table_col_pairs.map_with_index { |other_tc, other_index|
+        if other_index == index
+        # the current table
+          SelectTableAndColumns.new(tc.table_obj,
+                                    [tc.table_obj.index_name])
+        else
+          # other table: keep just the table, not the columns
+          SelectTableAndColumns.new(other_tc.table_obj, nil)
+        end
+      }
+      @index_tables[tc.table_obj.table_name] = execute_command(SQLQuery.select(index_table_col_pairs,
+                                                                               value_restrictions, parameters))
+    }
+
+    # map gold to something else?
+    # yes, if parameters[gold] has been set
+    if @parameters["gold"]
+      @map_gold = true
+      # remember which column in the DB table is the gold column
+      @gold_index = column_names.index(@parameters["gold"])
+    else
+      @map_gold = false
+    end
+  end
+
+  ################
+  # close
+  #
+  # to be called when the view is no longer needed:
+  # frees the DBResult objects underlying this view
+  def close
+    unless @view_empty
+      @main_table.free
+      @index_tables.each_value { |t| t.free }
+    end
+  end
+
+  ################
+  # write_to_file
+  #
+  # writes instances to a file
+  # each instance given as a comma-separated list of features
+  # The features are the ones given in my_feature_list
+  # (parameter to the new() method) above, in that order,
+  # plus (dynamic) gold, which is last.
+  #
+  # guarantees that comma is used only to separate features -- but no other
+  # changes in the feature values
+  def write_to_file(file, # stream to write to
+                    dyn_gold_id=nil) #string: ID of a DynGold object from the dynamic_feature_list.
+                                     # if nil, main gold is used
+
+    each_instance_s(dyn_gold_id) { |instance_string|
+      file.puts instance_string
+    }
+  end
+
+
+  ################
+  # each_instance_s
+  #
+  # yields each instance as a string:
+  # a comma-separated list of features
+  # The features are the ones given in my_feature_list
+  # (parameter to the new() method) above, in that order,
+  # plus (dynamic) gold, which is last.
+  #
+  # guarantees that comma is used only to separate features -- but no other
+  # changes in the feature values
+  def each_instance_s(dyn_gold_id=nil) #string: ID of a DynGold object from the dynamic_feature_list.
+                                     # if nil, main gold is used
+    each_array(dyn_gold_id) {|array|
+      yield array.map { |entry| entry.to_s.gsub(/,/, "COMMA") }.join(",")
+    }
+  end
+
+  ################
+  # each_hash
+  #
+  # iterates over hashes representing rows
+  #          in each row, there is a gold key/value pair
+  #          specified by the optional argument dyn_gold_id.
+  #          which is the string ID of a  DynGold object
+  #          from the dynamic_feature_list.
+  #          If arg is not present, main gold is used
+  #
+  #          The key for the gold is the dyn_gold_id
+  #          If that is nil, the key is 'gold'
+  #
+  # yields: hashes column_name -> column_value
+  def each_hash(dyn_gold_id=nil) #string: ID of a DynGold object from the dynamic_feature_list, or nil
+    if @view_empty
+      return
+    end
+    if @map_gold
+      dyn_gold_obj = fetch_dyn_gold_obj(dyn_gold_id)
+    end
+    @main_table.reset
+
+    @main_table.each_hash { |row_hash|
+      if @map_gold
+        row_hash[@parameters["gold"]] = dyn_gold_obj.make(row_hash[@parameters["gold"]])
+      end
+
+      yield row_hash
+    }
+  end
+
+  ################
+  # each_array
+  #
+  # iterates over arrays representing rows
+  #          the last item of each row is the gold column
+  #          selected by the optional argument dyn_gold_id.
+  #          which is the string ID of a  DynGold object
+  #          from the dynamic_feature_list.
+  #          If arg is not present, main gold is used
+  #
+  # yields: arrays of column values,
+  #         values are in the order of my_feature_list given
+  #         to the new() method, (dynamic) gold is last
+  def each_array(dyn_gold_id=nil) #string: ID of a DynGold object from the dynamic_feature_list, or nil
+
+    if @view_empty
+      return
+    end
+    if @map_gold
+      dyn_gold_obj = fetch_dyn_gold_obj(dyn_gold_id)
+    end
+    @main_table.reset
+
+    @main_table.each {|row|
+      if @gold_index
+        gold = row.delete_at(@gold_index)
+        if @map_gold
+          row.push dyn_gold_obj.make(gold)
+        else
+          row.push gold
+        end
+      end
+
+      yield row
+    }
+  end
+
+  ################
+  # update_column
+  #
+  # update a column for all rows of this view
+  #
+  # Given a column name to be updated, and a list of value tuples,
+  # update each row of the view, or rather the appropriate column of each row of the view,
+  # with values for that row.
+  #
+  # the list has the same length as the view, as there must be a value tuple
+  # for each row of the view.
+  #
+  # returns: nothing
+  def update_column(name, # string: column name
+                    values) # array of Objects
+
+    if @view_empty
+      raise "Cannot update empty view"
+    end
+
+    # find the first table in @table_col_pairs that has
+    # a column with this name
+    # and update that column
+    @table_col_pairs.each do |tc|
+      if (tc.columns.is_a?(Array) && tc.columns.include?(name)) || (tc.columns == "*" && tc.table_obj.list_column_names.include?(name))
+        table_name = tc.table_obj.table_name
+        # sanity check: number of update entries must match
+        # number of entries in this view
+        unless values.length == @index_tables[table_name].num_rows
+          $stderr.puts "Error: length of value array (#{values.length}) is not equal to length of view (#{@index_tables[table_name].num_rows})!"
+          exit 1
+        end
+
+        @index_tables[tc.table_obj.table_name].reset
+
+        values.each do |value|
+          index = @index_tables[table_name].fetch_row.first
+          tc.table_obj.update_row(index, [[name, value]])
+        end
+
+        return
+      end
+    end
+
+    # no match found
+    $stderr.puts "View.rb Error: cannot update a column that is not in this view: #{name}"
+    exit 1
+  end
+
+
+  ################
+  # each_sentence
+  #
+  # like each_hash, but it groups the row hashes sentence-wise
+  # sentence boundaries in the view are detected by the change in a
+  # special column describing sentence IDs
+  #
+  # also needs a dyngold object id
+  #
+  # returns: an array of hashes column_name -> column_value
+  def each_sentence(dyn_gold_id = nil)  # string: ID of a DynGold object from the dynamic_feature_list, or nil
+
+    # sanity check 1: need to know what the sentence ID is
+    unless @parameters["sentence_id_feature"]
+      raise "I need the name of the sentence ID feature for each_sentence"
+    end
+    # sanity check 2: the view needs to include the sentence ID
+    unless column_names.include? @parameters["sentence_id_feature"]
+      raise "View.each_sentence: Cannot do this without sentence ID in the view"
+    end
+
+    last_sent_id = nil
+    sentence = []
+    each_hash(dyn_gold_id) {|row_hash|
+      if last_sent_id != row_hash[@parameters["sentence_id_feature"]] and
+          (!(last_sent_id.nil?))
+        yield sentence
+        sentence = []
+      end
+      last_sent_id = row_hash[@parameters["sentence_id_feature"]]
+      sentence << row_hash
+    }
+    unless sentence.empty?
+      yield sentence
+    end
+  end
+
+  ######################
+  # length
+  #
+  # returns the length of the view: the number of its rows
+  def length
+    return @index_tables[@table_col_pairs.first.table_obj.table_name].num_rows
+  end
+
+  ###
+  private
+
+  ################
+  # column_names
+  #
+  # returns: array:string
+  #   the list of column names for this view
+  #   in the right order
+  def column_names
+    if @view_empty
+      return []
+    else
+      return @main_table.list_column_names
+    end
+  end
+
+  ######
+  # fetch_dyn_gold_obj
+  #
+  # given an ID of a gold object, look for the DynGold object
+  # with this ID in the dynamic_feature_list and return it
+  # If the ID is nil, use the standard dynamic gold ID that
+  # has been set in the new() method.
+  # If that is nil too, take the non-modified gold as a
+  # default: return a dummy object with a make() method
+  # that just returns its parameter.
+  #
+  # returns: object offering a make() method
+
+  def fetch_dyn_gold_obj(dyn_gold_id) # string or nil
+    # find a DynGold object that will transform the gold column
+    if dyn_gold_id.nil?
+      dyn_gold_id = @parameters["standard_dyngold_id"]
+    end
+
+    dyn_gold_obj = "we need an object that can do 'make'"
+    if dyn_gold_id
+      unless @parameters["dynamic_feature_list"]
+        raise "No dynamic features given"
+      end
+
+      dyn_gold_obj = @parameters["dynamic_feature_list"].detect { |obj|
+        obj.id == dyn_gold_id
+      }
+      if dyn_gold_obj.nil?
+        $stderr.puts "View.rb: Unknown DynGold ID " + dyn_gold_id
+        $stderr.puts "Using unchanged gold"
+        dyn_gold_id = nil
+      end
+    end
+
+    unless dyn_gold_id
+      # no dynamic gold ID: use unchanged gold by default
+      class << dyn_gold_obj
+        def make(x)
+          x
+        end
+        def id
+          return "gold"
+        end
+      end
+    end
+    return dyn_gold_obj
+  end
+
+  def execute_command(command)
+    begin
+      return @db_obj.query(command)
+    rescue MysqlError => e
+      $stderr.puts "Error executing SQL query. Command was:\n" + command
+      $stderr.puts "Error code: #{e.errno}"
+      $stderr.puts "Error message: #{e.error}"
+      raise e
+    end
+  end
+
+end