frprep 0.0.1.prealpha

data/lib/rosy/RosyPhase2FeatureExtractors.rb

@@ -0,0 +1,230 @@
+####
+# ke & sp 
+# adapted to new feature extractor class,
+# Collins and Tiger features combined:
+# SP November 2005
+#
+# Feature Extractors for Rosy, Phase 2
+#
+# These are features that are computed on the basis of the Phase 1 feature set
+#
+# This consists of all features which have to know feature values for other nodes
+# (e.g. am I the nearest node to the target?) or similar. 
+#
+# Contract: each feature extractor inherits from the RosyPhase2FeatureExtractor class
+#
+# Feature extractors return nil if no feature value could be returned
+
+
+# Salsa packages
+require 'rosy/AbstractFeatureAndExternal'
+require 'common/SalsaTigerRegXML'
+
+# Fred and Rosy packages
+require "common/RosyConventions"
+
+
+################################
+# base class for all following feature extractors
+
+class RosyPhase2FeatureExtractor < AbstractFeatureExtractor
+
+  ###
+  # we do not overwrite "train" and "refresh" --
+  # this is just for features which have to train external models on aspects of the data
+
+  ###
+  # returns a string: "phase 1" or "phase 2",
+  # depending on whether the feature is computed
+  # directly from the SalsaTigerSentence and the SynNode objects
+  # or whether it is computed from the phase 1 features
+  # computed for the training set
+  #
+  # Here: all features in this packages are phase 2
+  def RosyPhase2FeatureExtractor.phase()
+    return "phase 2"
+  end
+
+  ###
+  # returns an array of strings, providing information about
+  # the feature extractor
+  def RosyPhase2FeatureExtractor.info()
+    return super().concat(["rosy"])
+  end
+
+  ###
+  # set sentence, set node, set general settings: this is done prior to
+  # feature computation using compute_feature_value()
+  # such that computations that stay the same for
+  # several features can be done in advance
+  def RosyPhase2FeatureExtractor.set(var_hash)
+    @@split_nones = var_hash["split_nones"]
+    return true
+  end
+
+  # check if the current feature is computable, i.e. if all the necessary 
+  # Phase 1 features are in the present model..
+  def RosyPhase2FeatureExtractor.is_computable(given_extractor_list)
+    return (eval(self.name()).extractor_list - given_extractor_list).empty?
+  end
+  
+  # this probably has to be done for each feature:
+  # identify sentences and the target, and recombine into a large array  
+  def compute_features_on_view(view)
+    result = Array.new(eval(self.class.name()).feature_names.length)
+    result.each_index {|i|
+      result[i] = Array.new
+    }
+    view.each_sentence {|instance_features|
+      sentence_result = compute_features_for_sentence(instance_features)
+      if result.length != sentence_result.length
+        raise "Error: number of features computed for a sentence is wrong!"
+      else
+        result.each_index {|i|
+          if sentence_result[i].length != instance_features.length
+            raise "Error: number of feature values does not match number of sentence instances!"
+          end
+          result[i] += sentence_result[i]
+        }
+      end
+    }
+    return result
+  end
+
+  private
+
+  # list of all the Phase 1 extractors that a particular feature extractor presupposes
+  def RosyPhase2FeatureExtractor.extractor_list()
+    return []
+  end
+
+  # compute the feature values for all instances of one sentence
+  # left to be specified
+  # returns (see AbstractFeatureAndExternal) an array of columns (arrays)
+  # The length of the array corresponds to the number of features
+  def compute_features_for_sentence(instance_features) # array of hashes features -> values
+    raise "Overwrite me"
+  end
+
+
+end
+
+
+##############################################
+# Individual feature extractors
+##############################################
+
+####################
+# nearestNode
+#
+# compute whether if my head word is the nearest word to the target, 
+# according to some criterion
+
+class NearestNodeFeature < RosyPhase2FeatureExtractor
+  NearestNodeFeature.announce_me()
+  
+  def NearestNodeFeature.designator()
+    return "nearest_node"
+  end
+  def NearestNodeFeature.feature_names()
+    return ["nearest_pt_path",  # the nearest node with a specific pt_path                         
+            "neareststring_pt",# the nearest pt (string distance) 
+            "nearestpath_pt"]   # the nearest pt (path length) ]
+  end
+  def NearestNodeFeature.sql_type()
+    return "TINYINT"
+  end
+  def NearestNodeFeature.feature_type()
+    return "syn"
+  end
+
+  #####
+  private
+
+  def NearestNodeFeature.extractor_list()
+    return ["worddistance","pt_path","pt","path_length"]
+  end
+  
+  def compute_features_for_sentence(instance_features)
+    
+    # for each "interesting" feature, compute a hash map value -> index
+    # also compute a hashmap index -> distance
+    # so we efficiently compute, for each feature value, the index with min distance 
+    
+    dist_hash = Hash.new # node id -> word distance
+    pl_hash   = Hash.new # node id -> path length
+    path_hash = Hash.new # path -> node id array
+    pt_hash = Hash.new   # pt -> node id array
+    
+    result = [Array.new(instance_features.length),
+              Array.new(instance_features.length),
+              Array.new(instance_features.length)]
+    
+    instance_features.each_index {|inst_id|
+      instance_hash = instance_features[inst_id]
+      dist_hash[inst_id] = instance_hash["worddistance"]
+      pl_hash[inst_id] = instance_hash["path_length"]
+      
+      # record paths
+      pt_path = instance_hash["pt_path"]
+      unless path_hash.key? pt_path
+        path_hash[pt_path] = Array.new
+      end
+      path_hash[pt_path] << inst_id
+
+      # record pts
+      pt = instance_hash["pt"]
+      unless pt_hash.key? pt
+        pt_hash[pt] = Array.new
+      end
+      pt_hash[pt] << inst_id
+
+    }
+
+    # compute feature value for each instance of each path
+    # nearest-path feature is feature 0 of the extractor.
+    path_hash.each {|path,inst_ids|
+      distances = inst_ids.map {|inst_id| dist_hash[inst_id]}
+        min_dist = distances.min
+        inst_ids.each {|inst_id|
+          distance = dist_hash[inst_id]
+        if distance == min_dist and path != @exp.get("noval")
+          result[0][inst_id] = 1
+        else
+          result[0][inst_id] = 0
+        end
+      }
+    }
+
+    # nearest-pt (string dist) feature is feature 1 of the extractor
+    pt_hash.each{|pt,inst_ids|
+      distances = inst_ids.map {|inst_id| dist_hash[inst_id]}
+      min_dist = distances.min
+      inst_ids.each {|inst_id|
+        distance = dist_hash[inst_id]
+        if distance == min_dist and pt != @exp.get("noval")
+          result[1][inst_id] = 1
+        else
+          result[1][inst_id] = 0
+        end
+      }
+    } 
+    
+    # nearest-pt (path length) feature is feature 2 of the extractor
+    pt_hash.each{|pt,inst_ids|
+      path_lengths = inst_ids.map {|inst_id| pl_hash[inst_id]}
+      min_pl = path_lengths.min
+      inst_ids.each {|inst_id|
+        path_length = pl_hash[inst_id]
+        if path_length == min_pl and pt != @exp.get("noval")
+          result[2][inst_id] = 1
+        else
+          result[2][inst_id] = 0
+        end
+      }
+    } 
+
+    return result
+  end  
+end
+

data/lib/rosy/RosyPruning.rb

@@ -0,0 +1,165 @@
+######
+# XpPrune
+# Katrin Erk Jan 30, 2006
+#
+# Pruning for Rosy: mark constituents that as likely/unlikely to instantiate
+# a role.
+#
+# Pruning currently available:
+# Both Xue/Palmer original and a modified version for FrameNet
+
+require "common/ruby_class_extensions"
+
+require "rosy/RosyFeatureExtractors"
+require "common/RosyConventions"
+require "rosy/RosyConfigData"
+require "rosy/RosyIterator"
+
+###
+# Pruning, derived from the Xue/Palmer algorithm
+#
+# implemented in the Interpreter Class of each individual parser
+class PruneFeature < RosySingleFeatureExtractor
+  PruneFeature.announce_me()
+
+  def PruneFeature.feature_name()
+    return "prune"
+  end
+  def PruneFeature.sql_type()
+    return "TINYINT"
+  end
+  def PruneFeature.feature_type()
+    return "syn"
+  end
+  def PruneFeature.info()
+    # additional info: I am an index feature
+    return super().concat(["index"])
+  end
+
+  ################
+  private
+
+  def compute_feature_instanceOK()
+    retv = @@interpreter_class.prune?(@@node, @@paths, @@terminals_ordered)
+    if [0, 1].include? retv
+      return retv
+    else
+      return 0
+    end
+  end
+end
+
+####################
+# HIER changeme
+class TigerPruneFeature < RosySingleFeatureExtractor
+  TigerPruneFeature.announce_me()
+
+  def TigerPruneFeature.feature_name()
+    return "tiger_prune"
+  end
+  def TigerPruneFeature.sql_type()
+    return "TINYINT"
+  end
+  def TigerPruneFeature.feature_type()
+    return "syn"
+  end
+  def TigerPruneFeature.info()
+    # additional info: I am an index feature
+    return super().concat(["index"])
+  end
+
+  ################
+  private
+
+  def compute_feature_instanceOK()
+    if @@changeme_tiger_include.include? @@node
+      return 1
+    else
+      return 0
+    end
+  end
+end
+
+
+
+
+#######################3
+# Pruning: 
+# packaging all methods that will be needed to 
+# implement it, 
+# given that the xp_prune feature defined above
+# has been computed for each constituent during featurization.
+class Pruning
+
+  ###
+  # returns true if some kind of pruning has been set in the experiment file
+  #  else false
+  def Pruning.prune?(exp)  # Rosy experiment file object
+    if exp.get("prune")
+      return true
+    else
+      return false
+    end
+  end
+
+  ###
+  # returns: string, the name of the pruning column
+  #  nil if no pruning has been set
+  def Pruning.colname(exp)
+    if exp.get("prune")
+      return exp.get("prune")
+    else
+      return nil
+    end               
+  end
+
+  ###
+  # make ValueRestriction according to the pruning option set in 
+  # the experiment file:
+  #       WHERE <pruning_column_name> = 1
+  # where <pruning_column_name> is the name of one of the 
+  # pruning features defined above, the same name that has
+  # been set as the value of the pruning parameter in the experiment file
+  #
+  # return: ValueRestriction object (see RosyConventions)
+  #  If no pruning has been set in the experiment file, returns nil
+  def Pruning.restriction_removing_pruned(exp) # Rosy experiment file object
+    if (method = Pruning.colname(exp))
+      return ValueRestriction.new(method, 1)
+    else
+      return nil
+    end
+  end
+
+  ###
+  # given the name of a DB table column and an iterator that
+  # iterates over some data, 
+  # assuming that the column describes some classifier run results,
+  # choose all rows where the pruning column is 0 (i.e. all instances
+  # that have been pruned away) and set the value of the given column 
+  # to noval for them all, marking them as "not assigned any role".
+  def Pruning.integrate_pruning_into_run(run_column, # string: run column name
+                                         iterator,   # RosyIterator object
+                                         exp)        # Rosy experiment file object
+    unless Pruning.prune?(exp)
+      # no pruning activated
+      return
+    end
+                          
+    iterator.each_group { |group_descr_hash, group|
+      # get a view of all instances for which prune == 0, i.e. that have been pruned away
+      view = iterator.get_a_view_for_current_group(
+                                                   [run_column], 
+                                                   [ValueRestriction.new(Pruning.colname(exp), 0)]
+                                                   )
+      # make a list of column values that are all noval
+      all_noval = Array.new
+      view.each_instance_s { |inst|
+        all_noval << exp.get("noval")
+      }
+      # and set all selected instances to noval
+      view.update_column(run_column, all_noval)
+      view.close()
+    }
+  end
+end

data/lib/rosy/RosyServices.rb

@@ -0,0 +1,744 @@
+# RosyServices
+# KE May 05
+#
+# One of the main task modules of Rosy:
+# remove database tables and experiments,
+# dump experiment to files and load from files
+
+require "common/ruby_class_extensions"
+
+# Rosy packages
+require "common/RosyConventions"
+require "rosy/RosyIterator"
+require "rosy/RosySplit"
+require "rosy/RosyTask"
+require "rosy/RosyTrainingTestTable"
+require "rosy/View"
+
+# Frprep packages
+require "common/FrPrepConfigData"
+
+###################################################
+class RosyServices < RosyTask
+
+  def initialize(exp,      # RosyConfigData object: experiment description
+		 opts,     # hash: runtime argument option (string) -> value (string)
+		 ttt_obj)  # RosyTrainingTestTable object
+
+    ##
+    # remember the experiment description
+
+    @exp = exp
+    @ttt_obj = ttt_obj
+
+    ##
+    # check runtime options
+
+    @tasks = Array.new
+    # defaults:
+    @step = "onestep"
+    @splitID = nil
+    @testID = default_test_ID()
+
+
+    opts.each do |opt,arg|
+      case opt
+      when "--deltable", "--delexp", "--delruns", "--delsplit", "--deltables"
+        #####
+        # In enduser mode, you cannot delete things
+        in_enduser_mode_unavailable()
+	@tasks << [opt, arg]
+      when "--dump", "--load", "--writefeatures"
+	@tasks << [opt, arg]
+      when "--step"
+	unless ["argrec", "arglab", "both", "onestep"].include? arg
+	  raise "Classification step must be one of: argrec, arglab, both, onestep. I got: " + arg.to_s
+	end
+	@step = arg
+
+      when "--logID"
+        @splitID = arg
+
+      when "--testID"
+        @testID = arg
+
+      else
+	# this is an option that is okay but has already been read and used by rosy.rb
+      end	
+    end
+    # announce the task
+    $stderr.puts "---------"
+    $stderr.puts "Rosy experiment #{@exp.get("experiment_ID")}: Services."
+    $stderr.puts "---------"
+  end
+
+  #####
+  # perform
+  #
+  # do each of the inspection tasks set as options
+  def perform()
+    @tasks.each { |opt, arg|
+      case opt
+      when "--deltable"
+        del_table(arg)
+      when "--deltables"
+        del_tables()
+      when "--delexp"
+        del_experiment()
+      when "--delruns"
+        del_runs()
+      when "--delsplit"
+        del_split(arg)
+      when "--dump"
+        dump_experiment(arg)
+      when "--load"
+        load_experiment(arg)
+      when "--writefeatures"
+ 	write_features(arg)
+      end
+    }
+  end
+
+  ################################
+  private
+  
+  #####
+  # del_table
+  #
+  # remove one DB table specified by its name
+  # The method verifies whether the table should be deleted.
+  # If the user gives an answer starting in "y", the table is deleted.
+  def del_table(table_name) # string: name of DB table
+    # check if we have this table
+    unless @ttt_obj.database.list_tables().include? table_name
+      $stderr.puts "Cannot find DB table #{table_name}."
+      return
+    end
+
+    # really delete?
+    $stderr.print "Really delete DB table #{table_name}? [y/n] "
+    answer = gets().chomp()
+    unless answer =~ /^y/
+      return
+    end
+
+    begin
+      @ttt_obj.database.drop_table(table_name)
+    rescue
+      $stderr.puts "Error: Removal of #{table_name} failed."
+      return
+    end
+
+    # done.
+    $stderr.puts "Deleted table #{table_name}."
+  end
+
+  ######
+  # del_tables
+  #
+  # for all the tables in the database, present their name and size,
+  # and ask if it should be deleted.
+  # this is good for cleaning up!
+  
+  def del_tables() 
+    @ttt_obj.database.list_tables().each { |table_name|
+
+      STDERR.print "Delete table #{table_name} (num. rows #{@ttt_obj.database.num_rows(table_name)})? [y/n] "
+      answer = gets().chomp()
+
+      if answer =~ /^y/
+        deletion_worked = false
+        begin
+          @ttt_obj.database.drop_table(table_name)
+          deletion_worked = true
+        rescue
+          deletion_worked = false
+        end
+        if deletion_worked 
+          STDERR.puts "Table #{name} removed."
+        else
+          $stderr.puts "Error: Removal of #{name} failed."          
+        end
+      end
+    }
+  end
+
+  #####
+  # del_experiment
+  #
+  # remove the experiment described by the experiment file @exp
+  # The method verifies whether the experiment should be deleted.
+  # If the user gives an answer starting in "y", the experiment is deleted.
+  def del_experiment()
+    data_dir = File.new_dir(@exp.instantiate("rosy_dir", "exp_ID" => @exp.get("experiment_ID")))
+
+    # no data? then don't do anything
+    if not(@ttt_obj.train_table_exists?) and
+        @ttt_obj.testIDs().empty? and
+        @ttt_obj.splitIDs().empty? and
+        Dir[data_dir + "*"].empty?
+      $stderr.puts "No data to delete for experiment #{@exp.get("experiment_ID")}."
+      # we have just made the directory data_dir by calling @exp.new_dir
+      # undo that
+      %x{rmdir #{data_dir}}
+      return
+    end       
+        
+
+    # really delete?
+    $stderr.print "Really delete experiment #{@exp.get("experiment_ID")}? [y/n] "
+    answer = gets().chomp()
+    unless answer =~ /^y/
+      return
+    end
+
+    # remove main table
+    @ttt_obj.remove_train_table()
+
+    # remove test tables
+    @ttt_obj.testIDs.each { |testID|
+      @ttt_obj.remove_test_table(testID)
+    }
+
+
+    # remove split tables
+    @ttt_obj.splitIDs.each { |splitID|
+      @ttt_obj.remove_split_table(splitID, "train")
+      @ttt_obj.remove_split_table(splitID, "test")
+    }
+
+    # remove files
+    %x{rm -rf #{data_dir}}
+
+    # done.
+    $stderr.puts "Deleted experiment #{@exp.get("experiment_ID")}."
+  end
+
+  ############
+  # del_runs
+  #
+  # interactively remove runs from the current experiment
+  def del_runs()
+    # iterate through all tables and runs
+    @ttt_obj.runlog_to_s_list().each { |table_descr|
+      unless table_descr["runlist"].empty?
+        # print description of the table
+        $stderr.puts table_descr["header"]
+        
+        table_descr["runlist"].each { |run_id, run_descr|
+          $stderr.puts run_descr
+          $stderr.puts "Delete this run? [y/n] "
+          answer = gets().chomp()
+          if answer =~ /^[yY]/
+            @ttt_obj.delete_runlog(table_descr["table_name"], run_id)
+          end
+        }
+      end
+    }
+  end
+
+  ##############
+  # del_split
+  # 
+  # remove the split with the given ID
+  # from the current experiment:
+  # delete split tables, remove from list of test and split tables
+  def del_split(splitID)
+    # does the split exist?
+    unless @ttt_obj.splitIDs.include? splitID
+      $stderr.puts "del_split:"
+      $stderr.puts "Sorry, I don't have a split with ID #{splitID} in experiment #{exp.get("experiment_ID")}."
+      return
+    end
+
+    # really delete?
+    $stderr.print "Really delete split #{splitID} of experiment #{@exp.get("experiment_ID")}? [y/n] "
+    answer = gets().chomp()
+    unless answer =~ /^y/
+      return
+    end
+
+    # remove split tables
+    @ttt_obj.remove_split_table(splitID, "train")
+    @ttt_obj.remove_split_table(splitID, "test")
+
+    # remove classifiers for split
+    ["argrec", "arglab", "onestep"].each { |step|
+      classif_dir = classifier_directory_name(@exp,step, splitID)
+      %x{rm -rf #{classif_dir}}
+    }
+  end
+
+  ##############
+  # write features to files:
+  # use
+  #  @step, @testID, @splitID to determine feature set to write
+  def write_features(directory) # string: directory to write to, may be nil
+
+    ###
+    # prepare directory to write to
+    if directory != ""
+      # the user has given a directory.
+      # make sure it ends in /
+      dir = File.new_dir(directory)
+    else
+      # use the default directory: <rosy_dir>/tables
+      dir = File.new_dir(@exp.instantiate("rosy_dir", 
+                                          "exp_ID" => @exp.get("experiment_ID")),
+                         "your_feature_files")
+    end
+    $stderr.puts "Writing feature files to directory " + dir
+
+    ##
+    # check: if this is about a split, do we have it?
+    if @splitID
+      unless @ttt_obj.splitIDs().include?(@splitID)
+        $stderr.puts "Sorry, I have no data for split ID #{@splitID}."
+        exit 1
+      end
+    end
+
+    ##
+    # inform the user on what we are writing
+    if @splitID
+      $stderr.puts "Writing data according to split '#{@splitID}'"
+    elsif @testID
+      # do we have this test set? else write only training set
+      if @ttt_obj.testIDs().include?(@testID)
+	$stderr.puts "Writing training data, and test data with ID '#{@testID}'"
+      else
+        $stderr.puts "Warning: no data for test ID '#{@testID}', writing only training data."
+	@testID = nil
+      end
+    end
+    
+    $stderr.puts "Writing data for classification step '#{@step}'."
+    $stderr.puts
+
+    ##
+    # write training data
+    $stderr.puts "Writing training sets"
+    iterator = RosyIterator.new(@ttt_obj, @exp, "train", 
+				"step" => @step, 
+				"splitID" => @splitID,
+				"prune" => true)
+
+    # get the list of relevant features,
+    # remove the features that describe the unit by which we train, 
+    # since they are going to be constant throughout the training file
+    features = @ttt_obj.feature_info.get_model_features(@step) - 
+      iterator.get_xwise_column_names()
+
+    # but add the gold feature
+    unless features.include? "gold"
+      features << "gold"
+    end
+
+
+    write_features_aux(dir, "training", @step, iterator, features)
+
+    ## 
+    # write test data
+    if @testID
+      $stderr.puts "Writing test sets"
+      filename = dir + "test.data"
+      iterator = RosyIterator.new(@ttt_obj, @exp, "test", 
+                                  "step" => @step, 
+                                  "testID" => @testID, 
+                                  "splitID" => @splitID,
+                                  "prune" => true)
+      write_features_aux(dir, "test", @step, iterator, features)
+    end
+  end
+
+  ########
+  # write_features_aux: actually do the writing
+  def write_features_aux(dir,      # string: directory to write to
+			 dataset,  # string: training or test
+			 step,     # string: argrec, arglab, onestep
+			 iterator, # RosyIterator tuned to what we're writing
+			 features) # array:string: list of features to include in views
+
+    # proceed one group at a time
+    iterator.each_group { |group_descr_hash, group|
+      # get data for this group
+      view = iterator.get_a_view_for_current_group(features)
+      
+      #filename: e.g. directory/training.Statement.data
+      filename = dir + dataset + "." + 
+	step + "." + 
+	group.gsub(/\s/, "_") + ".data"
+
+      begin
+	file = File.new(filename, "w")
+      rescue
+	$stderr.puts "Error: Could not write to file #{filename}, exiting."
+	exit 1
+      end
+
+      view.each_instance_s { |instance_string|
+	# change punctuation to _PUNCT_
+	# and change empty space to _
+	# because otherwise some classifiers may spit
+	file.puts prepare_output_for_classifiers(instance_string)
+      }
+      file.close()
+      view.close()
+    }
+  end
+  
+  ##############3
+  # dump_experiment
+  #
+  # dump to file:
+  # - main table. filename: main
+  # - test tables. filename: test.<testID>
+  # - split tables. filenames: split.train.<ID>, split.test.<ID>
+  # of the experiment given in @exp.
+  #
+  # Each table is dumped in a separate file:
+  # The first line describes column names,
+  # each following line is one row of the DB.
+  #
+  # Files are written to <rosy_dir>/tables
+  def dump_experiment(directory) #string: directory to write to, may be nil
+    ###
+    # prepare:
+
+    # directory to write to
+    if directory != ""
+      # the user has given a directory.
+      # make sure it ends in /
+      dir = File.new_dir(directory)
+    else
+      # use the default directory: <rosy_dir>/tables
+      dir = File.new_dir(@exp.instantiate("rosy_dir", 
+                                          "exp_ID" => @exp.get("experiment_ID")),
+                         "tables")
+    end
+    $stderr.puts "Writing experiment data to directory " + dir
+
+    ###
+    # dump main table
+    
+    $stderr.puts "Dumping main table"
+    filename = dir + "main"
+    begin
+      file = File.new(filename, "w")
+    rescue
+      $stderr.puts "Sorry, couldn't write to #{filename}"
+      return
+    end
+
+    if @ttt_obj.train_table_exists?
+      iterator = RosyIterator.new(@ttt_obj, @exp, "train", "xwise" => "frame")
+      table_obj = @ttt_obj.existing_train_table()
+      aux_dump(iterator, file, table_obj)
+    end
+
+    ###
+    # dump test tables
+    
+    unless @ttt_obj.testIDs.empty?
+      $stderr.print "Dumping test tables: "
+    end
+    @ttt_obj.testIDs.each { |testID|
+
+      filename = dir + "test." + testID
+      $stderr.print filename, " "
+      begin
+        file = File.new(filename, "w")
+      rescue
+        $stderr.puts "Sorry, couldn't write to #{filename}"
+        return
+      end
+      
+      if @ttt_obj.test_table_exists?(testID)
+        iterator = RosyIterator.new(@ttt_obj, @exp, "test", "testID" => testID, "xwise" => "frame")
+        table_obj = @ttt_obj.existing_test_table(testID)
+        aux_dump(iterator, file, table_obj)
+      end
+    }
+    unless @ttt_obj.testIDs.empty?
+      $stderr.puts
+    end
+
+    # dump split tables
+    unless @ttt_obj.splitIDs.empty?
+      $stderr.print "Dumping split tables: "
+    end
+    @ttt_obj.splitIDs.each { |splitID|
+      ["train", "test"].each { |dataset|
+        
+        filename = dir + "split." + dataset + "." + splitID
+        $stderr.print filename, " "
+        begin
+          file = File.new(filename, "w")
+        rescue
+          $stderr.puts "Sorry, couldn't write to #{filename}"
+          return
+        end
+
+        if @ttt_obj.split_table_exists?(splitID, dataset)
+          iterator = RosyIterator.new(@ttt_obj, @exp, dataset, "splitID" => splitID, "xwise" => "frame")
+          table_obj = @ttt_obj.existing_split_table(splitID, dataset, RosySplit.split_index_colname())
+          aux_dump(iterator, file, table_obj)
+        end
+      }
+    }
+    unless @ttt_obj.splitIDs.empty?
+      $stderr.puts
+    end
+
+    ###
+    # dump classification run logs
+    @ttt_obj.to_file(dir)    
+  end
+
+  ################3
+  # aux_dump
+  #
+  # auxiliary method for dump_experiment()
+  def aux_dump(iterator, # RosyIterator object, refers to table to write
+               file, # stream: write to this file
+               table_obj) # DB table to be written
+    
+    # write all columns except the autoincrement index
+    # columns_to_write: array:string*string column name, column SQL type
+    columns_to_write = Array.new()
+    @ttt_obj.database.list_column_formats(table_obj.table_name).each { |column_name, column_type|
+      unless column_name == table_obj.index_name
+        # check: when loading we make assumptions on the field types that can happen.
+        # check here that we don't get any unexpected field types
+        case column_type
+        when /^varchar\d*\(\d+\)$/i, /^char\d*\(\d+\)$/i, /^tinyint(\(\d+\))*$/i, /^int/i
+        else
+          $stderr.puts "Problem with SQL type #{column_type} of column #{column_name}:"
+          $stderr.puts "Won't be able to handle it when loading."
+        end
+        columns_to_write << [column_name, column_type]
+      end
+    }
+    columns_as_array = columns_to_write.map { |name, type| name}
+    
+    # write column names and types
+    file.puts columns_to_write.map { |name, type| name }.join(",")
+    file.puts columns_to_write.map { |name, type| type }.join(",")
+    
+    # access groups and write data
+    
+    iterator.each_group { |hash, framename|
+      view = iterator.get_a_view_for_current_group(columns_as_array)
+
+      # write instances
+      view.each_hash { |instance|
+        file.puts columns_to_write.map { |name, type| 
+          # get column entries in order of column names
+          instance[name] 
+        }.map { |entry|
+          # remove commas
+          entry.to_s.gsub(/,/, "COMMA") 
+        }.join(",")
+      }
+      view.close()
+    }
+  end
+
+  ##############3
+  # load_experiment
+  #
+  # load from file:
+  # - main table
+  # - test tables
+  # - split tables
+  #
+  # Filenames: see dump_experiment()
+  #
+  # Data is loaded into the current experiment,
+  # previous experiment data is removed
+  #
+  # Each table is loaded from a separate file:
+  # The first line describes column names,
+  # each following line is one row of the DB.
+  def load_experiment(directory) # string: directory to read from, may be nil
+
+    ###
+    # ask whether this is what the user intended
+    $stderr.puts "Load experiment data from files into the current experiment:"
+    $stderr.puts "This will overwrite existing data of experiment #{@exp.get("experiment_ID")}."
+    $stderr.print "Proceed? [y/n] "
+    answer = gets().chomp()
+    unless answer =~ /^y/
+      return
+    end
+
+    ##
+    # adjoin preprocessing experiment file to find out about the language of the data
+    # for this it is irrelevant whether we take the training or test
+    # preprocessing experiment file. Take the training file.
+    preproc_expname = @exp.get("preproc_descr_file_train")
+    if not(preproc_expname)
+      $stderr.puts "Please set the name of the preprocessing exp. file name"
+      $stderr.puts "in the experiment file, parameter preproc_descr_file_train."
+      exit 1
+    elsif not(File.readable?(preproc_expname))
+      $stderr.puts "Error in the experiment file:"
+      $stderr.puts "Parameter preproc_descr_file_train has to be a readable file."
+      exit 1
+    end
+    preproc_exp = FrPrepConfigData.new(preproc_expname)
+    @exp.adjoin(preproc_exp)
+
+    ###
+    # read the data where?
+    if directory != ""
+      # the user has given a directory
+      # make sure it exists
+      dir = File.existing_dir(directory)
+    else
+      # default: <rosy_dir>/tables
+      dir = File.existing_dir(@exp.instantiate("rosy_dir", 
+                                               "exp_ID" => @exp.get("experiment_ID")), 
+                              "tables")
+    end
+    $stderr.puts "Reading experiment data from directory " + dir
+
+    ###
+    # read tables
+    Dir.foreach(dir) { |filename|
+      case filename
+      when "main"
+        # read main file
+        $stderr.puts "Writing main DB table"
+
+        file = File.new(dir + filename)
+        col_names, col_types = aux_read_colnames(file, @ttt_obj.feature_names)
+
+        # start new main table, removing the old
+        table_obj = @ttt_obj.new_train_table()
+        # write file contents to the DB table
+        aux_transfer_to_table(file, table_obj, col_names, col_types)
+
+      when /^test\.(.+)$/
+        # read test file
+        testID = $1
+        $stderr.puts "Writing test DB table with ID #{testID}"
+
+        file = File.new(dir + filename)
+        col_names, col_types = aux_read_colnames(file, @ttt_obj.feature_names)
+
+        # start new test table, removing the old
+        table_obj = @ttt_obj.new_test_table(testID)
+        # write file contents to the DB table
+        aux_transfer_to_table(file, table_obj, col_names, col_types)
+
+      when /^split\.(train|test)\.(.+)$/
+        dataset = $1
+        splitID = $2
+        $stderr.puts "Writing split #{dataset} DB table with ID #{splitID}"
+
+        file = File.new(dir + filename)
+        col_names, col_types = aux_read_colnames(file, nil)
+        table_obj = @ttt_obj.new_split_table(splitID, dataset, RosySplit.split_index_colname())
+        # write file contents to the DB table
+        aux_transfer_to_table(file, table_obj, col_names, col_types)
+
+      else
+        # not a filename we recognize
+        # don't do anything with it        
+      end
+    }
+
+    success = @ttt_obj.from_file(dir)
+    unless success
+      $stderr.puts "Could not read previous classification runs, assume empty."
+    end
+  end
+
+  ##
+  # aux_read_colnames
+  #
+  # auxiliary method for load_experiment
+  #
+  # read column names from dumped DB table file,
+  # compare to given set of column names,
+  # complain if they don't match
+  #
+  # returns: array*array, first array(strings): column names
+  #   second array(strings): column SQL types
+  def aux_read_colnames(file, # stream: file to read DB table info from
+                        exp_colnames) # array:string, column names defined in the experiment file
+    colnames = aux_read_columns(file)
+    # sanity check: features here the same as in the experiment file?
+    if exp_colnames
+      feature_colnames = colnames.select { |c| c !~ /^#{@exp.get("classif_column_name")}/ }
+      unless feature_colnames.sort() == exp_colnames.sort()
+        raise "Feature name mismatch!\nIn the experiment file, you have specified:\n" +
+            exp_colnames.sort().join(",") +
+            "\nIn the table I'm reading from file I got:\n" +
+            feature_colnames.sort().join(",")
+      end
+    else
+      # no check of column name match requested
+    end
+    coltypes = aux_read_columns(file)
+    return [colnames, coltypes]
+  end
+                        
+
+  ##
+  # aux_transfer_columns
+  # 
+  # auxiliary method for load_experiment:
+  # read a line from file, split it at commas
+  #   to arrive at the contents
+  def aux_read_columns(file) # stream: file
+    line = file.gets()
+    if line.nil?
+      return nil
+    end
+    line.chomp!
+    return line.split(",")
+  end
+
+  ###
+  # aux_transfer_to_table
+  #
+  # auxiliary method for load_experiment:
+  # read columns from file,
+  # write to table, omitting nil values
+  def aux_transfer_to_table(file, # stream: read from this file
+                            table_obj, # DBTable object: write to this table
+                            col_names, # array:string: these are the column names
+                            col_types) # array:string: SQL column types
+
+
+    # sp workaround Tue Aug 23
+    # table may have too few classification columns since it has been created with only
+    # the standard set of classification columns. Add more if needed
+
+    col_names.each {|col_name|
+      if !(table_obj.list_column_names.include? col_name) and col_name =~ /^#{@exp.get("classif_column_name")}/
+        table_obj.change_format_add_columns([[col_name, "VARCHAR(20)"]])
+      end
+    }
+
+    # write file contents to the DB table
+    names_and_values = Array.new
+    while row =  aux_read_columns(file)
+      names_and_values.clear()
+      col_names.each_with_index { |name, ix|
+        unless row[ix].nil?
+          if col_types[ix] =~ /^(TINYINT|tinyint)/ 
+            # integer value: map!
+            names_and_values << [name, row[ix].to_i]
+          else
+            # string value: leave as is
+            names_and_values << [name, row[ix]]
+          end
+        end
+      }
+      table_obj.insert_row(names_and_values)
+    end
+  end
+ end