shalmaneser-lib 1.2.rc5

data/lib/configuration/rosy_config_data.rb

@@ -0,0 +1,126 @@
+require_relative 'config_data'
+require_relative 'configuration_error'
+
+##############################
+# Class RosyConfigData
+#
+# inherits from ConfigData,
+# sets features for ROSY
+module Shalmaneser
+  module Configuration
+    class RosyConfigData < ConfigData
+      CONFIG_DEFS = {
+        "feature" => "list",
+        "classifier" => "list",
+        "verbose" => "bool",
+        "experiment_ID" => "string",
+        "directory_input_train" => "string",
+        "directory_input_test" => "string",
+        "directory_output" => "string",
+        "preproc_descr_file_train" => "string",
+        "preproc_descr_file_test" => "string",
+        "external_descr_file"    => "string",
+        "dbtype" => "string",    # "mysql" or "sqlite"
+
+        "host" => "string",      # DB access: sqlite only
+        "user" => "string",
+        "passwd" => "string",
+        "dbname" => "string",
+
+        "data_dir" => "string",  # for external use
+        "rosy_dir" => "pattern", # for internal use only, set by rosy.rb
+
+        "classifier_dir" => "string", # if present, special directory for classifiers
+
+        "classif_column_name" => "string",
+        "main_table_name" => "pattern",
+        "test_table_name" => "pattern",
+
+        "eval_file" => "pattern",
+        "log_file" => "pattern",
+        "failed_file" => "pattern",
+        "classifier_file" => "pattern",
+        "classifier_output_file" => "pattern",
+        "noval" => "string",
+        "split_nones" => "bool",
+        "print_eval_log" => "bool",
+        "assume_argrec_perfect" => "bool",
+        "xwise_argrec" => "string",
+        "xwise_arglab" => "string",
+        "xwise_onestep" => "string",
+        "fe_syn_repair" => "bool", # map words to constituents for FEs: idealize?
+        "fe_rel_repair" => "bool", # FEs: include non-included relative clauses into FEs
+        "prune" => "string", # pruning prior to argrec?
+
+        # Imported from PrepConfigData
+        'do_postag' => 'bool',
+        'do_lemmatize' => 'bool',
+        'do_parse' => 'bool',
+        'pos_tagger' => 'string',
+        'lemmatizer' => 'string',
+        'parser' => 'string'
+      }
+
+      def initialize(filename)
+        super(filename, CONFIG_DEFS, ["exp_ID", "test_ID", "split_ID",
+                                      "feature_name", "classif", "step",
+                                      "group", "dataset", "mode"])
+
+        # set access functions for list features
+        set_list_feature_access("feature",
+                                method("access_feature"))
+
+        # set access functions for list features
+        set_list_feature_access("classifier",
+                                method("access_feature"))
+        validate
+      end
+
+      ###
+      # protected
+
+      #####
+      # access_feature
+      #
+      # access function for feature 'feature'
+      #
+      # assumed format in the config file:
+      #
+      #   feature = path [option]*
+      #
+      # i.e. first the name of the feature type to use, then
+      # optionally options associated with that feature,
+      # e.g. 'argrec': use that feature only when computing argrec
+      #
+      # the access function is called with parameter val_list, an array of
+      # string tuples, one string tuple for each feature defined.
+      # the first string in the tuple is the feature name, the rest are the options
+      #
+      # returns: a list of pairs [feature_name(string), options(array:string)]
+      # of defined features
+      def access_feature(val_list) # array:array:string: list of tuples defined in config file
+        # for feature 'feature'
+        if val_list.nil?
+          []
+        else
+          val_list.map do |feature_descr_tuple|
+            [feature_descr_tuple.first, feature_descr_tuple[1..-1]]
+          end
+        end
+      end
+
+      private
+
+      def validate
+        msg = []
+
+        unless get("experiment_ID") =~ /^[A-Za-z0-9_]+$/
+          msg << 'Please choose an alphanumeric experiment ID! '\
+                 "You provided: #{get('experiment_ID')}"
+        end
+
+        raise(ConfigurationError, msg.join("\n")) if msg.any?
+      end
+    end
+  end
+end

data/lib/db/db_interface.rb

@@ -0,0 +1,50 @@
+# DBInterface
+#
+# Okay, things are getting somewhat complicated here with all
+# the DB classes, but this is how it all fits together:
+#
+# - DBWrapper: abstract class describing the DB interface
+# - DBMySQL, DBSQLite: subclasses of DBWrapper, for MySQL
+#     and SQLite, respectively
+# - DBInterface: class to be used from outside,
+#     decides ( based on the experiment file) whether to use
+#     MySQL or SQLite and makes an object of the right kind,
+#     'require'-ing either DBMySQL or DBSQLite, but not both,
+#     because the right ruby packages might not be installed
+#     for both SQL systems
+# @note This class will be obsolete if we deleten MySQL.
+class DBInterface
+
+  def self.get_db_interface(exp, # experiment file object with 'dbtype' entry
+                            dir = nil, # string: Shalmaneser directory (used by SQLite only)
+                            identifier = nil) # string: identifier of the data (SQLite)
+
+    case exp.get('dbtype')
+    when 'mysql'
+      begin
+        require 'db/db_mysql'
+      rescue => e
+        p e
+        STDERR.puts 'Error loading DB interface.'
+        STDERR.puts 'Make sure you have the Ruby MySQL package installed.'
+        exit 1
+      end
+
+      return DBMySQL.new(exp)
+    when 'sqlite'
+      begin
+        require 'db/db_sqlite'
+      rescue
+        STDERR.puts 'Error loading DB interface.'
+        STDERR.puts 'Make sure you have the Ruby SQLite package installed.'
+        exit 1
+      end
+      return DBSQLite.new(exp, dir, identifier)
+
+    else
+      STDERR.puts 'Error: database type needs to be either "mysql" or "sqlite"".'
+      STDERR.puts 'Please set parameter "dbtype" in the experiment file accordingly.'
+      exit 1
+    end
+  end
+end

data/lib/db/db_mysql.rb

@@ -0,0 +1,141 @@
+# DBMysql: a subclass of DBWrapper.
+#
+# Use a MySQL server to access a database.
+# Use the Ruby mysql interface package for that.
+
+require 'mysql'
+require 'db/db_wrapper'
+
+#################
+class DBMySQLResult < DBResult
+  # initialize with the result of Mysql::query
+  # which is a MysqlResult object
+  #
+  # also remember the offset of the first row
+  # for reset()
+  def initialize(value)
+    super(value)
+    @row_first = @result.row_tell
+  end
+
+  ###
+  # reset object such that each() can be run again
+  def reset
+    @result.row_seek(@row_first)
+  end
+
+  ###
+  # column names: list of strings
+  def list_column_names
+    current = @result.row_tell
+    fields = @result.fetch_fields.map(&:name)
+    @result.row_seek(current)
+
+    fields
+  end
+end
+
+#################
+class DBMySQL < DBWrapper
+  ###
+  # initialization:
+  #
+  # open connection to MySQL server
+  def initialize(exp)  # RosyConfigData experiment file object
+    super(exp)
+
+    @database = Mysql.real_connect(@exp.get('host'), @exp.get('user'),
+                                   @exp.get('passwd'), @exp.get('dbname'))
+
+  end
+
+
+  ###
+  # make a table
+  #
+  # returns: nothing
+  def create_table(table_name, # string
+                   column_formats, # array: array: string*string [column_name,column_format]
+                   index_column_names, # array: string: column_name
+                   indexname)  # string: name of automatically created index column
+
+    string = "CREATE TABLE #{table_name} (" +
+      "#{indexname} INT NOT NULL AUTO_INCREMENT"
+
+    # column declarations
+    unless column_formats.empty?
+      string << ", "
+      string << column_formats.map { |name, format| name.to_s + " " + format.to_s }.join(",")
+    end
+
+    # primary key
+    string << ", " + "PRIMARY KEY(#{indexname})"
+
+    # other keys
+    unless index_column_names.empty?
+      string << ", "
+      string << index_column_names.map { |name| "KEY(#{name})" }.join(",")
+    end
+    string << ");"
+
+    query_noretv(string)
+  end
+
+  ####
+  # querying the database:
+  # returns a DBResult object
+  def query(query)
+    result = @database.query(query)
+    if result
+      return DBMySQLResult.new(result)
+    else
+      return nil
+    end
+  end
+
+  ####
+  # querying the database:
+  # no result value
+  def query_noretv(query)
+    @database.query(query)
+    return nil
+  end
+
+  ###
+  # list all tables in the database
+  #
+  # array of strings
+  def list_tables
+    return @database.list_tables
+  end
+
+
+  #####
+  # list_column_formats
+  #
+  # list column names and column types of this table
+  #
+  # returns: array:string*string, list of pairs [column name, column format]
+  def list_column_formats(table_name)
+    retv = []
+    @database.query("DESCRIBE #{table_name}").each_hash { |field|
+      retv << [field["Field"], field["Type"]]
+    }
+    return retv
+  end
+
+  ####
+  # num_rows
+  #
+  # determine the number of rows in a table
+  # returns: integer or nil
+  def num_rows(table_name)
+    @database.query("SHOW TABLE STATUS").each_hash { |hash|
+      if hash["Name"] == table_name
+        return hash["Rows"]
+      end
+    }
+    return nil
+  end
+
+end

data/lib/db/db_sqlite.rb

@@ -0,0 +1,280 @@
+# DBSQLite: a subclass of DBWrapper.
+#
+# Use SQLite to access a database.
+# Use the Ruby sqlite3 interface package for that.
+
+require 'sqlite3'
+require 'tempfile'
+
+require 'db/db_wrapper'
+
+#################
+class DBSQLiteResult < DBResult
+  # initialize with the result of SQLite::execute()
+  # which returns an array of rows
+  # Each row is an array
+  # but additionally has attributes
+  # - fields: returns an array of strings, the column names
+  # - types: returns an array of strings, the column types
+  def initialize(value)
+    super(value)
+    @counter = 0
+  end
+
+  ###
+  # column names: list of strings
+  def list_column_names
+    return @result.columns
+  end
+
+  # number of rows: returns an integer
+  def num_rows
+    # remember where we were in iterating over items
+    tmp_counter = @counter
+
+    # reset, and iterate over all rows to count
+    reset
+    retv = 0
+    each { |x| retv += 1}
+
+    # return to where we were in iterating over items
+    reset
+    while @counter < tmp_counter
+      @result.next
+      @counter += 1
+    end
+
+    # and return the number of rows
+    return retv
+  end
+
+
+  # yields each row as an array of values
+  def each
+    @result.each { |row|
+      @counter += 1
+      yield row.map { |x| x.to_s }
+    }
+  end
+
+  # yields each row as a hash: column name=> column value
+  def each_hash
+    @result.each { |row|
+      @counter += 1
+
+      row_hash = {}
+      row.fields.each_with_index { |key, index|
+        row_hash[key] = row[index].to_s
+      }
+      yield row_hash
+    }
+  end
+
+
+  ###
+  # reset such that each() can be run again on the result object
+  def reset
+    @result.reset
+    @counter = 0
+  end
+
+  # free object
+  def free
+    @result.close
+  end
+
+  # returns row as an array of column contents
+  def fetch_row
+    @counter += 1
+    return @result.next
+  end
+end
+
+#################
+class DBSQLite < DBWrapper
+
+  ###
+  # initialization:
+  #
+  # open database file according to the given identifier
+  def initialize(exp,  # RosyConfigData experiment file object
+                 dir = nil,  # string: directory for Shalmaneser internal data, ends in "/"
+                 identifier = nil) # string: identifier to use for the database
+    super(exp)
+
+    # dir and identifier may be nil, if we're only opening this object
+    # in order to make temp databases
+    if dir and identifier
+      @database = SQLite3::Database.new(dir + identifier.to_s + ".db")
+    else
+      @database = nil
+    end
+
+    # temp file for temp database
+    @tf = nil
+  end
+
+  ###
+  # make a table
+  #
+  # returns: nothing
+  def create_table(table_name, # string
+                   column_formats, # array: array: string*string [column_name,column_format]
+                   index_column_names, # array: string: column_name
+                   indexname)  # string: name of automatically created index column
+
+    # primary key and auto-increment column
+    string = "CREATE TABLE #{table_name} (" +
+             "#{indexname} INTEGER PRIMARY KEY"
+
+    # column declarations
+    unless column_formats.empty?
+      string << ", "
+      string << column_formats.map { |name, format|
+        # include other keys
+        if index_column_names.include? name
+          name.to_s + " KEY " + format.to_s
+        else
+          name.to_s + " " + format.to_s
+        end
+      }.join(",")
+    end
+    string << ");"
+
+    query_noretv(string)
+  end
+
+  ###
+  # remove a table
+  def drop_table(table_name)
+    query_noretv("DROP TABLE " + table_name)
+  end
+
+  ###
+  def query(query)
+    if @database
+      return DBSQLiteResult.new(@database.query(query))
+    else
+      return nil
+    end
+  end
+
+  ####
+  # querying the database:
+  # no result value
+  def query_noretv(query)
+    if @database
+      @database.execute(query)
+    end
+    return nil
+  end
+
+  ###
+  # list all tables in the database
+  #
+  # array of strings
+  def list_tables
+    if @database
+      return @database.execute("select name from sqlite_master;").map { |t|
+        t.to_s
+      }
+    else
+      return nil
+    end
+  end
+
+  #####
+  # list_column_formats
+  #
+  # list column names and column types of this table
+  #
+  # returns: array:string*string, list of pairs [column name, column format]
+  def list_column_formats(table_name)
+    unless @database
+      return nil
+    end
+
+    table_descr = @database.execute("select * from sqlite_master where name=='#{table_name}';")
+    # this is an array of pieces of table description.
+    # the piece in the column called 'sql' is the 'create' statement.
+    # get the 'create' statement
+    begin
+      field_names = table_descr[0].fields
+    rescue
+      $stderr.puts "SQLite error: could not read description of table #{table_name}"
+      exit 1
+    end
+    create_index = (0..field_names.length).detect { |ix| field_names[ix] == 'sql' }
+
+    # try to parse column names out of the 'create' statement
+    if table_descr[0][create_index] =~ /^\s*create table \S+\s*\((.*)\)\s*$/i
+      # we now have something of shape ' a key varchar2(30), b varchar2(30)'
+      # split at the comma, remove whitespace at beginning and end
+      # then split again to get pairs [column name, column format]
+      return $1.split(",").map { |col_descrip|
+        pieces = col_descrip.strip.split.reject { |entry|
+          entry =~ /^key$/i or  entry =~ /^primary$/i
+        }
+        if pieces.length > 2
+          $stderr.puts "Warning: problematic column format in #{col_descrip}, may be parsed wrong."
+        end
+        pieces
+      }
+    else
+      $stderr.puts "SQLite error: cannot read column names"
+      exit 1
+    end
+  end
+
+  ####
+  # num_rows
+  #
+  # determine the number of rows in a table
+  # returns: integer or nil
+  def num_rows(table_name)
+    unless @database
+      return nil
+    end
+
+    rows_s = @database.get_first_value( "select count(*) from #{table_name}" )
+    if rows_s
+      return rows_s.to_i
+    else
+      return nil
+    end
+  end
+
+  ####
+  # make a temporary table: make a table in a new, temporary file
+  #
+  # returns: DBWrapper object (or object of current subclass)
+  # that has the @table_name attribute set to the name of a temporary DB
+  #
+  # same as in superclass
+  #
+  #   def make_temp_table(column_formats, # array: string*string [column_name,column_format]
+  #                       index_column_names, # array: string: column_name
+  #                       indexname)  # string: name of autoincrement primary index
+
+  #     temp_obj = self.clone()
+  #     temp.initialize_temp_table(column_formats, index_column_names, indexname)
+  #     return temp_obj
+  #   end
+
+  def drop_temp_table
+    @tf.close(true)
+    @database = nil
+  end
+
+  ##############################
+  protected
+
+  def initialize_temp_table(column_formats, index_column_names, indexname)
+    @table_name = "temptable"
+    @tf = Tempfile.new("temp_table")
+    @tf.close
+    @database = SQLite3::Database.new(@tf.path)
+    create_table(@table_name, column_formats, index_column_names, indexname)
+  end
+
+end