shalmaneser-rosy 1.2.0.rc4 → 1.2.rc5

data/lib/rosy/RosyFeaturize.rb

@@ -5,28 +5,26 @@
 # featurize data and store it in the database
 
 # Salsa packages
-require "common/SynInterfaces"
-require "common/ruby_class_extensions"
+require 'external_systems'
+require "ruby_class_extensions"
 
-# Frprep packages
-#require "common/prep_config_data"
-
-# Rosy packages
 require "rosy/FailedParses"
 require "rosy/FeatureInfo"
 require "rosy/InputData"
-require "rosy/rosy_config_data"
-require "common/RosyConventions"
+require 'configuration/rosy_config_data'
+require 'rosy/rosy_conventions'
 require "rosy/RosySplit"
 require "rosy/RosyTask"
 require "rosy/RosyTrainingTestTable"
-require "rosy/View"
+# require "rosy/View"
 
+module Shalmaneser
+module Rosy
 class RosyFeaturize < RosyTask
 
   def initialize(exp,      # RosyConfigData object: experiment description
-		 opts,     # hash: runtime argument option (string) -> value (string)
-		 ttt_obj)  # RosyTrainingTestTable object
+                 opts,     # hash: runtime argument option (string) -> value (string)
+                 ttt_obj)  # RosyTrainingTestTable object
 
     ##
     # remember the experiment description
@@ -34,32 +32,23 @@ class RosyFeaturize < RosyTask
     @exp = exp
     @ttt_obj = ttt_obj
 
-    ##
-    # check runtime options
-    if $ENDUSER_MODE
-      @dataset = "test"
-    else
-      @dataset = nil
-    end
-    @testID = default_test_ID()
+    @testID = ::Shalmaneser::Rosy.default_test_ID
     @splitID = nil
     @append_rather_than_overwrite = false
 
     opts.each do |opt,arg|
       case opt
       when "--dataset"
-	unless ["train", "test"].include? arg
-	  raise "--dataset needs to be either 'train' or 'test'"
-	end
-	@dataset = arg
+        unless ["train", "test"].include? arg
+          raise "--dataset needs to be either 'train' or 'test'"
+        end
+        @dataset = arg
       when "--logID"
         @splitID = arg
       when "--testID"
-	@testID = arg
+        @testID = arg
       when "--append"
         @append_rather_than_overwrite = true
-      else
-	# this is an option that is okay but has already been read and used by rosy.rb
       end
     end
 
@@ -69,11 +58,6 @@ class RosyFeaturize < RosyTask
       exit 1
     end
 
-    #####
-    # Enduser mode: featurization only of test data
-    in_enduser_mode_ensure(@dataset == "test")
-    in_enduser_mode_ensure(@append_rather_than_overwrite == false)
-
     # announce the task
     $stderr.puts "---------"
     $stderr.puts "Rosy experiment #{@exp.get("experiment_ID")}: Featurization of dataset #{@dataset}"
@@ -98,12 +82,12 @@ class RosyFeaturize < RosyTask
     #   $stderr.puts "Parameter #{preproc_parameter} has to be a readable file."
     #   exit 1
     # end
-    # preproc_exp = FrPrepConfigData.new(preproc_expname)
+    # preproc_exp = FrappeConfigData.new(preproc_expname)
     # @exp.adjoin(preproc_exp)
 
     ###
     # find appropriate class for interpreting syntactic structures
-    @interpreter_class = SynInterfaces.get_interpreter_according_to_exp(@exp)
+    @interpreter_class = ::Shalmaneser::ExternalSystems.get_interpreter_according_to_exp(@exp)
 
     ###
     # prepare featurization
@@ -111,7 +95,7 @@ class RosyFeaturize < RosyTask
       unless @exp.get("directory_input_" + @dataset)
         raise "Please set 'directory_input_train' and/or 'directory_input_test' in your experiment file."
       end
-      prepare_main_featurization(File.existing_dir(@exp.get("directory_input_" + @dataset)), 
+      prepare_main_featurization(File.existing_dir(@exp.get("directory_input_" + @dataset)),
                                  @testID)
     end
   end
@@ -120,16 +104,16 @@ class RosyFeaturize < RosyTask
   # perform
   #
   # compute features and write them to the DB table
-  def perform()
+  def perform
     if @dataset
       # compute features for main or test table
-      perform_main_featurization()
+      perform_main_featurization
     end
   end
 
   #####################
   private
-  
+
   ###
   # prepare_main_featurization
   #
@@ -141,9 +125,9 @@ class RosyFeaturize < RosyTask
                                  testID)  # string: name of this testset, or nil for no testset
 
     # sanity check
-    unless datapath 
+    unless datapath
       raise "No input path given in the preprocessing experiment file.\n" +
-	"Please set 'directory_preprocessed there."
+        "Please set 'directory_preprocessed there."
     end
     unless File.exists? datapath and File.directory? datapath
       raise "I cannot read the input path " + datapath
@@ -151,7 +135,7 @@ class RosyFeaturize < RosyTask
 
     ##
     # determine features and feature formats
-     
+
     # create feature extraction wrapper object
     @input_obj = InputData.new(@exp, @dataset, @ttt_obj.feature_info, @interpreter_class, datapath)
 
@@ -181,23 +165,23 @@ class RosyFeaturize < RosyTask
 
 
       if @append_rather_than_overwrite
-        # add to existing DB table	
-	@db_table = @ttt_obj.existing_train_table()
+        # add to existing DB table
+        @db_table = @ttt_obj.existing_train_table
 
       else
         # start new DB table
-        @db_table = @ttt_obj.new_train_table()
+        @db_table = @ttt_obj.new_train_table
       end
 
     when "test"
 
       if @append_rather_than_overwrite
         # add to existing DB table
-	@db_table = @ttt_obj.existing_test_table(testID)
+        @db_table = @ttt_obj.existing_test_table(testID)
 
       else
         # start new DB table
-	@db_table = @ttt_obj.new_test_table(testID)
+        @db_table = @ttt_obj.new_test_table(testID)
 
       end
 
@@ -217,65 +201,68 @@ class RosyFeaturize < RosyTask
 
     ###########
     # write state to log
-    log_filename = 
+    log_filename =
        File.new_filename(@exp.instantiate("rosy_dir",
                                           "exp_ID" => @exp.get("experiment_ID")),
                          "featurize.log")
-    
+
     ##############
     # input object, compute features for **PHASE 1*:
     #
     # make features for each instance:
     # features that can be computed from this instance alone
-    
-    `echo "[#{Time.now().to_s}] Featurize: Start phase 1 feature extraction" >> #{log_filename}`
-    
+
+    # @todo AB: Change this to my logger!
+    `echo "[#{Time.now.to_s}] Featurize: Start phase 1 feature extraction" >> #{log_filename}`
+
     @input_obj.each_instance_phase1 { |feature_list| # list of pairs [column_name(string), value(whatever)]
 
       # write instance to @db_table
       @db_table.insert_row(feature_list)
     }
 
-    # during featurisation, an Object with info about failed parses has been created 
+    # during featurisation, an Object with info about failed parses has been created
     # now get this object and store it in a file in the datadir
-    
-    failed_parses_obj = @input_obj.get_failed_parses()
-    
-    failed_parses_filename = 
+
+    failed_parses_obj = @input_obj.get_failed_parses
+
+    failed_parses_filename =
       File.new_filename(@exp.instantiate("rosy_dir",
-                                         "exp_ID" => @exp.get("experiment_ID")), 
+                                         "exp_ID" => @exp.get("experiment_ID")),
                         @exp.instantiate("failed_file",
                                          "exp_ID" => @exp.get("experiment_ID"),
                                          "split_ID" => "none",
                                          "dataset" => "none"))
-                        
+
     failed_parses_obj.save(failed_parses_filename)
-    
+
     ################
     # input object, compute features for **PHASE 2**:
     #
     # based on all features from Phase 1, make additional features
-    
-    `echo "[#{Time.now().to_s}] Featurize: Start phase 2 feature extraction" >> #{log_filename}`
 
-    iterator = RosyIterator.new(@ttt_obj, @exp, @dataset, 
+    `echo "[#{Time.now.to_s}] Featurize: Start phase 2 feature extraction" >> #{log_filename}`
+
+    iterator = RosyIterator.new(@ttt_obj, @exp, @dataset,
                                 "testID" => @testID,
                                 "splitID" => @splitID,
                                 "xwise" => "frame")
     iterator.each_group { |dummy1, dummy2|
       view = iterator.get_a_view_for_current_group("*")
-      
+
       @input_obj.each_phase2_column(view) { |feature_name, feature_values|
         view.update_column(feature_name, feature_values)
       }
-      
-      view.close()      
+
+      view.close
     }
-    
+
     #########
     # finished!!
     #
-    `echo "[#{Time.now().to_s}] Featurize: Finished" >> #{log_filename}`    
-    
+    `echo "[#{Time.now.to_s}] Featurize: Finished" >> #{log_filename}`
+
   end
 end
+end
+end

data/lib/rosy/RosyInspect.rb

@@ -5,20 +5,20 @@
 # inspect global data and experiment-specific data of the system
 
 # Rosy packages
-require "common/RosyConventions"
+# require "RosyConventions"
+require 'db/select_table_and_columns'
 require "rosy/RosySplit"
 require "rosy/RosyTask"
 require "rosy/RosyTrainingTestTable"
-require "rosy/View"
-
-# Frprep packages
-require "common/prep_config_data"
+require 'db/db_view'
 
+module Shalmaneser
+module Rosy
 class RosyInspect < RosyTask
 
   def initialize(exp,      # RosyConfigData object: experiment description
-		 opts,     # hash: runtime argument option (string) -> value (string)
-		 ttt_obj)  # RosyTrainingTestTable object
+                 opts,     # hash: runtime argument option (string) -> value (string)
+                 ttt_obj)  # RosyTrainingTestTable object
 
     ##
     # remember the experiment description
@@ -29,18 +29,18 @@ class RosyInspect < RosyTask
     ##
     # check runtime options
 
-    @tasks = Array.new
+    @tasks = []
     @test_id = nil
 
     opts.each do |opt,arg|
       case opt
       when "--tables", "--tablecont", "--runs", "--split"
-	@tasks << [opt, arg]
+        @tasks << [opt, arg]
       when "--testID"
-	@test_id = arg
+        @test_id = arg
       else
-	# this is an option that is okay but has already been read and used by rosy.rb
-      end	
+        # this is an option that is okay but has already been read and used by rosy.rb
+      end
     end
 
     ##
@@ -65,7 +65,7 @@ class RosyInspect < RosyTask
 #       $stderr.puts "Parameter #{preproc_parameter} has to be a readable file."
 #       exit 1
 #     end
-#     preproc_exp = FrPrepConfigData.new(preproc_expname)
+#     preproc_exp = FrappeConfigData.new(preproc_expname)
 #     @exp.adjoin(preproc_exp)
 
     # announce the task
@@ -78,22 +78,22 @@ class RosyInspect < RosyTask
   # perform
   #
   # do each of the inspection tasks set as options
-  def perform()
+  def perform
     @tasks.each { |opt, arg|
       case opt
       when "--tables"
-	inspect_tables()
+        inspect_tables
       when "--tablecont"
-	inspect_tablecont(arg)
+        inspect_tablecont(arg)
       when "--runs"
-        inspect_runs()
+        inspect_runs
       when "--split"
         inspect_split(arg)
       end
     }
 
     if @tasks.empty?
-      inspect_experiment()
+      inspect_experiment
     end
   end
 
@@ -103,24 +103,24 @@ class RosyInspect < RosyTask
   # print to stdout:
   # name and column names of each table
   # in this database
-  def inspect_tables()
+  def inspect_tables
     puts
     puts "-----------------------------------------------"
     puts "List of all tables in the database"
     puts "-----------------------------------------------"
     puts
 
-    @ttt_obj.database.list_tables().each { | table_name|
+    @ttt_obj.database.list_tables.each { | table_name|
       puts "Table " + table_name
       puts "\tColumns: "
       print "\t"
       count = 0
       @ttt_obj.database.list_column_formats(table_name).each { |column_name, column_format|
-	count += 1
-	print column_name, " (", column_format, ")\t"
-	if count % 4 == 0
-	  print "\n\t"
-	end
+        count += 1
+        print column_name, " (", column_format, ")\t"
+        if count % 4 == 0
+          print "\n\t"
+        end
       }
       puts
       puts
@@ -149,7 +149,7 @@ class RosyInspect < RosyTask
           # both table ID and number of lines
           # last part: number of lines. Rest: table ID
           # (re-join in case the table ID includes a ':')
-          num_lines = parts.pop()
+          num_lines = parts.pop
           table_id = parts.join(":")
         end
       elsif not(id_numlines.empty?)
@@ -159,38 +159,38 @@ class RosyInspect < RosyTask
     end
 
     # sanity check: existing table ID?
-    if table_id and not(@ttt_obj.database.list_tables().include?(table_id))
+    if table_id and not(@ttt_obj.database.list_tables.include?(table_id))
       $stderr.puts "Error: I don't know a table with ID #{table_id}"
       return
     end
-  
+
     if table_id
       # handle table with given table ID
 
-      puts 
+      puts
       puts "-----------------------------------------------"
       puts "Experiment " + @exp.get("experiment_ID").to_s + " table "+ table_id
       puts "-----------------------------------------------"
       puts
-      
+
       db_table = DBTable.new(@ttt_obj.database,
                              table_id,
                              "open",
                              "addcol_prefix" => @exp.get("classif_column_name"))
-      
+
       inspect_tablecont_aux(db_table, num_lines)
-      
+
     else
 
       # handle training data
-      puts 
+      puts
       puts "-----------------------------------------------"
       puts "Experiment " + @exp.get("experiment_ID").to_s + " training data"
       puts "-----------------------------------------------"
       puts
 
       if @ttt_obj.train_table_exists?
-        db_table = @ttt_obj.existing_train_table()
+        db_table = @ttt_obj.existing_train_table
         inspect_tablecont_aux(db_table, num_lines)
       else
         $stderr.puts "(No main table.)"
@@ -198,8 +198,8 @@ class RosyInspect < RosyTask
 
       # handle test data
       if @test_id
-        
-        puts 
+
+        puts
         puts "-----------------------------------------------"
         puts "Experiment " + @exp.get("experiment_ID").to_s + " test data (#{@test_id})"
         puts "-----------------------------------------------"
@@ -231,40 +231,40 @@ class RosyInspect < RosyTask
     print column_names.map { |n| "[" + n + "]" }.join(" ")
     puts
     puts
-    
+
     # select rows to print
     view = DBView.new([SelectTableAndColumns.new(table_obj, column_names)],
-		      [],        # no restrictions on rows to pick
-		      @ttt_obj.database, # database access
-		      "gold" => "gold",    # name of gold feature
-		      "line_limit" => num_lines) # number of lines to read
-    
+                      [],        # no restrictions on rows to pick
+                      @ttt_obj.database, # database access
+                      "gold" => "gold",    # name of gold feature
+                      "line_limit" => num_lines) # number of lines to read
+
     # and print them
     view.write_to_file($stdout)
-    view.close()
+    view.close
   end
 
   # print to stdout: all classification runs for the current experiment ID
-  def inspect_runs()
-    puts @ttt_obj.runlog_to_s()
+  def inspect_runs
+    puts @ttt_obj.runlog_to_s
   end
 
   # print to stdout: train, test sentence ID for given split
   def inspect_split(splitID)
 
-    puts 
+    puts
     puts "-----------------------------------------------"
     puts "Split " + splitID.to_s
     puts "-----------------------------------------------"
     puts
 
     ["train", "test"].each { |dataset|
-      
+
       puts "Dataset " + dataset
       puts "==========="
       puts
 
-      table = @ttt_obj.existing_split_table(splitID, dataset, RosySplit.split_index_colname())
+      table = @ttt_obj.existing_split_table(splitID, dataset, RosySplit.split_index_colname)
       view = DBView.new([SelectTableAndColumns.new(table, "*")], [], @ttt_obj.database)
       index = 1
       view.each_array { |row|
@@ -278,7 +278,7 @@ class RosyInspect < RosyTask
     }
   end
 
-  def inspect_experiment()
+  def inspect_experiment
     puts "------------------------------------"
     puts "Experiment #{@exp.get("experiment_ID").to_s}"
     puts "------------------------------------"
@@ -317,7 +317,7 @@ class RosyInspect < RosyTask
     puts "Classifier runs for this experiment:"
     puts "-----------------------"
     puts
-    puts @ttt_obj.runlog_to_s()
+    puts @ttt_obj.runlog_to_s
     puts
   end
 
@@ -334,3 +334,5 @@ class RosyInspect < RosyTask
   end
 
  end
+end
+end