shalmaneser-rosy 1.2.0.rc4 → 1.2.rc5

data/lib/rosy/GfInduceFeature.rb

@@ -5,61 +5,60 @@
 # feature for Rosy
 
 require "rosy/GfInduce"
-require "rosy/AbstractFeatureAndExternal"
-require "common/ruby_class_extensions"
-
-###
-# make filename for GfInduce picle file
-def filename_gfmap(exp,         # ExternalConfigData object
-		   interpreter) # SynInterpreter class
-
-  # output dir as given in my experiment file
-  # If there is an experiment ID, make subdirectory
-  # named after the experiment ID and place the data there.
-  output_dir = File.new_dir(exp.get("directory"))
-  if exp.get("experiment_id")
-    output_dir = File.new_dir(output_dir, exp.get("experiment_id"))
-  end
-  
-  # output file name: 
-  # Gfmap.{<service>=<system_name>.}*{OPT<service>=<system_name>.}*pkl
-  return output_dir + 
-    "Gfmap." + 
-    interpreter.systems().to_a.map { |service, system_name|
-    service.to_s+ "=" + system_name.to_s
-  }.sort.join(".") + "." +
-    interpreter.optional_systems().to_a.map { |service, system_name|
-    "OPT" + service.to_s + "=" + system_name.to_s
-  }.sort.join(".") + ".pkl"
-end
+require 'rosy/external_feature_extractor'
+require 'monkey_patching/file'
 
+module Shalmaneser
+module Rosy
 ################################
 # base class for all following feature extractors
 class GfInduceFeatureExtractor < ExternalFeatureExtractor
-  GfInduceFeatureExtractor.announce_me()
+  GfInduceFeatureExtractor.announce_me
 
   @@okay = true  # external experiment file present?
   @@gf_obj = nil # GfInduce object
   @@node_to_gf = nil # Hash: SynNodes of a sentence -> Gf label
 
-  def GfInduceFeatureExtractor.designator()
-    return "gf_fn"
+  ###
+  # Initialize: read GFInduce pickle
+  def initialize(exp,                  # experiment file object
+                 interpreter_class)    # SynInterpreter class
+
+    super(exp, interpreter_class)
+
+    if @exp_external
+      pickle_filename = filename_gfmap(@exp_external, @@interpreter_class)
+      @@gf_obj = GfInduce.from_file(pickle_filename)
+      @@okay = true
+
+    else
+      # signal that you cannot compute anything
+      @@okay = false
+    end
+  end
+
+  def self.designator
+    "gf_fn"
   end
-  def GfInduceFeatureExtractor.feature_names()
-    return ["gf_fn"]
+
+  def self.feature_names
+    ["gf_fn"]
   end
-  def GfInduceFeatureExtractor.sql_type()
-    return "VARCHAR(25)"
+
+  def self.sql_type
+    "VARCHAR(25)"
   end
-  def GfInduceFeatureExtractor.feature_type()
-    return "syn"
+
+  def self.feature_type
+    "syn"
   end
-  def GfInduceFeatureExtractor.phase()
-    return "phase 1"
+
+  def self.phase
+    "phase 1"
   end
 
   ###
-  # set sentence, set node, set other settings: 
+  # set sentence, set node, set other settings:
   # this is done prior to
   # feature computation using compute_feature()
   # such that computations that stay the same for
@@ -68,9 +67,9 @@ class GfInduceFeatureExtractor < ExternalFeatureExtractor
   # This is just relevant for Phase 1
   #
   # returns: false/nil if there was a problem
-  def GfInduceFeatureExtractor.set_sentence(sent,  # SalsaTigerSentence object
-                                            frame) # FrameNode object
-
+  # @param sent [SalsaTigerSentence]
+  # @param frame [FrameNode]
+  def self.set_sentence(sent, frame)
     super(sent, frame)
 
     if @@okay
@@ -78,7 +77,7 @@ class GfInduceFeatureExtractor < ExternalFeatureExtractor
 
       # let the GF object compute all subcat frames
       # for the target of this frame
-      subcatframes_of_current_target = @@gf_obj.apply(frame.target.children())
+      subcatframes_of_current_target = @@gf_obj.apply(frame.target.children)
 
       # keep the most frequent one of the
       # subcat frames returned by the GF object:
@@ -86,7 +85,7 @@ class GfInduceFeatureExtractor < ExternalFeatureExtractor
         # no subcat frames returned
         subcatframe = []
       else
-        # we have at least one subcat frame: 
+        # we have at least one subcat frame:
         # keep the most frequent one of them
         #
         # Also, subcatframes_of_current_target
@@ -98,9 +97,9 @@ class GfInduceFeatureExtractor < ExternalFeatureExtractor
           b.last <=> a.last
         }.first[1]
       end
-      
+
       # change into a mapping node(SynNode) -> GF(string)
-      @@node_to_gf = Hash.new
+      @@node_to_gf = {}
       subcatframe.each { |gf, prep, fe, synnodes|
         synnodes.each { |node|
           @@node_to_gf[node] = "#{gf} #{prep}"
@@ -109,25 +108,6 @@ class GfInduceFeatureExtractor < ExternalFeatureExtractor
     end
   end
 
-
-  ###
-  # Initialize: read GFInduce pickle
-  def initialize(exp,                  # experiment file object
-                 interpreter_class)    # SynInterpreter class
-
-    super(exp, interpreter_class)
-    
-    if @exp_external
-      pickle_filename = filename_gfmap(@exp_external, @@interpreter_class)
-      @@gf_obj = GfInduce.from_file(pickle_filename)
-      @@okay = true
-
-    else
-      # signal that you cannot compute anything
-      @@okay = false
-    end
-  end
-
   ###
   # compute: compute features
   #
@@ -135,14 +115,52 @@ class GfInduceFeatureExtractor < ExternalFeatureExtractor
   # length of feature_names()
   #
   # here: array of length one, content either a string or nil
-  def compute_features()
+  def compute_features
     # current node: @@node
     # check whether the current node has been assigned a slot
     # in the subcat frame
     if @@okay
-      return [ @@node_to_gf[@@node] ]
+      return [@@node_to_gf[@@node]]
     else
-      return [ nil ]
+      return [nil]
     end
   end
+
+  private
+
+  ###
+  # make filename for GfInduce picle file
+  # @param exp [ExternalConfigData]
+  # @param interpreter [SynInterpreter]
+  def filename_gfmap(exp, interpreter)
+    # output dir as given in my experiment file
+    # If there is an experiment ID, make subdirectory
+    # named after the experiment ID and place the data there.
+    output_dir = File.new_dir(exp.get("directory"))
+
+    if exp.get("experiment_id")
+      output_dir = File.new_dir(output_dir, exp.get("experiment_id"))
+    end
+
+    # output file name:
+    # Gfmap.{<service>=<system_name>.}*{OPT<service>=<system_name>.}*pkl
+    output_dir = output_dir + 'Gfmap.' + interpreter.systems.to_a
+
+    output_dir = output_dir.map do |service, system_name|
+      "#{service}=#{system_name}"
+    end
+
+    output_dir = output_dir.sort.join('.') + '.' +
+                 interpreter.optional_systems.to_a
+
+    output_dir = output_dir.map do |service, system_name|
+      "OPT#{service}=#{system_name}"
+    end
+
+    output_dir = output_dir.sort.join('.') + '.pkl'
+
+    output_dir
+  end
+end
+end
 end

data/lib/rosy/InputData.rb

@@ -7,37 +7,41 @@
 # featurization
 
 # Salsa packages
-require "common/Parser"
-require "common/SalsaTigerRegXML"
-require "common/ruby_class_extensions"
+require 'salsa_tiger_xml/file_parts_parser'
+# require "SalsaTigerRegXML"
+require 'salsa_tiger_xml/salsa_tiger_sentence'
+require "ruby_class_extensions"
 
 # Fred/Rosy packages
 require "rosy/FailedParses"
-require "common/RosyConventions"
+require 'rosy/rosy_conventions'
 require "rosy/RosyFeatureExtractors"
 require "rosy/RosyPhase2FeatureExtractors"
 require "rosy/RosyPruning"
 require "rosy/GfInduceFeature"
-require "common/FixSynSemMapping"
+require 'frappe/fix_syn_sem_mapping'
 
+module Shalmaneser
+module Rosy
 class InputData
 
   ###
   def initialize(exp_object,          # RosyConfigData object
                  dataset,             # train/test
-		 feature_info_object, # FeatureInfo object
+                 feature_info_object, # FeatureInfo object
                  interpreter_class,   # SynInterpreter class
-                 input_dir)           # Directory with input files   
+                 input_dir)           # Directory with input files
 
     @exp = exp_object
     @dataset = dataset
     @interpreter_class = interpreter_class
+    raise 'BumBamBim!!!' if @interpreter_class.nil?
     @input_dir = input_dir
     # store information about failed parses here
-    @failed_parses = FailedParses.new()
+    @failed_parses = FailedParses.new
 
     # feature_extractors_phase1: array of AbstractFeatureExtractor objects
-    @extractors_p1_rosy, @extractors_p1_other = feature_info_object.get_extractor_objects("phase 1", 
+    @extractors_p1_rosy, @extractors_p1_other = feature_info_object.get_extractor_objects("phase 1",
                                                                                           @interpreter_class)
 
     # global settings
@@ -47,18 +51,18 @@ class InputData
 
 #     # nothing to set here for now, so deactivated
 #     @extractors_p1_other.each { |extractor_obj|
-#       unless extractor_obj.class.set()
+#       unless extractor_obj.class.set
 #         raise "Some grave problem during feature extractor initialization"
 #       end
 #     }
 
 
     # feature_extractors_phase2: array of  AbstractFeatureExtractor objects
-    extractors_p2_rosy, extractors_p2_other = feature_info_object.get_extractor_objects("phase 2", 
+    extractors_p2_rosy, extractors_p2_other = feature_info_object.get_extractor_objects("phase 2",
                                                                                         @interpreter_class)
     @feature_extractors_phase2 = extractors_p2_rosy + extractors_p2_other
   end
-  
+
   ###
   # each_instance_phase1()
   #
@@ -68,14 +72,14 @@ class InputData
   # and yields one feature vector per instance
   #
   # yields: pairs [feature_name(string), feature_value(object)]
-  
-  def each_instance_phase1()
+
+  def each_instance_phase1
     Dir[@input_dir+"*.xml"]. each {|parsefilename|
 
-      xmlFile = FilePartsParser.new(parsefilename)
+      xmlFile = STXML::FilePartsParser.new(parsefilename)
       $stderr.puts "Processing #{parsefilename}"
       xmlFile.scan_s {|sent_string|
-        sent = SalsaTigerSentence.new(sent_string)
+        sent = STXML::SalsaTigerSentence.new(sent_string)
 
         # preprocessing: possibly change the SalsaTigerSentence object
         # before featurization
@@ -105,9 +109,9 @@ class InputData
           if skip_frame
             next
           end
-               
+
           sent.each_syn_node { |syn_node|
-            
+
             # Tell feature extractors about the current node:
             # first Rosy feature extractors, then the others
             # if there is a problem, skip this node
@@ -126,34 +130,34 @@ class InputData
             end
 
             # features: array of pairs: [feature_name(string), feature_value(object)]
-            features = Array.new
+            features = []
             (@extractors_p1_rosy + @extractors_p1_other).each { |extractor|
               # compute features
-              feature_names = extractor.class.feature_names()
+              feature_names = extractor.class.feature_names
               feature_index = 0
-              
+
               # append new features to features array
-              features.concat extractor.compute_features().map { |feature_value|
+              features.concat extractor.compute_features.map { |feature_value|
                 feature_name = feature_names[feature_index]
                 feature_index += 1
-                
+
                 # sanity check: feature value longer than the allotted space in the DB?
                 check_feature_length(feature_name, feature_value, extractor)
 
-                [feature_name, nonnil_feature(feature_value, extractor.class.sql_type()) ]
+                [feature_name, nonnil_feature(feature_value, extractor.class.sql_type) ]
               }
-            }            
+            }
             yield features
           } # each syn node
         } # each frame
       } # each sentence
     }
   end
-  
+
   ###
   # each_phase2_column
   #
-  # This method implements the application of the 
+  # This method implements the application of the
   # phase 2 extractors to data.
   #
   # Given a database view (of either training or test data),
@@ -169,10 +173,10 @@ class InputData
       feature_columns = extractor.compute_features_on_view(view)
       # interleave with feature values and yield
       feature_index = 0
-      feature_names = extractor.class.feature_names()
+      feature_names = extractor.class.feature_names
       feature_columns.each { |feature_values|
         yield [
-          feature_names[feature_index], 
+          feature_names[feature_index],
           feature_values.map { |feature_val| nonnil_feature(feature_val, extractor.class.sql_type)  }
         ]
         feature_index += 1
@@ -184,22 +188,18 @@ class InputData
   # get_failed_parses
   #
   # returns the FailedParses object in which the info about failed parses has been stored
-  def get_failed_parses()
-    return @failed_parses
+  def get_failed_parses
+    @failed_parses
   end
 
-  #################################
   private
 
-
   ###
-  def nonnil_feature(feature_value,
-                     sql_type)
-
+  def nonnil_feature(feature_value, sql_type)
     # feature value nil? then change to noval
-    if feature_value.nil? and sql_type =~ /CHAR/
+    if feature_value.nil? && sql_type =~ /CHAR/
       return @exp.get("noval")
-    elsif feature_value.class.to_s == "String" and feature_value.empty?
+    elsif feature_value.is_a?(String) && feature_value.empty?
       return @exp.get("noval")
     elsif feature_value.nil?
       return 0
@@ -209,12 +209,14 @@ class InputData
   end
 
   ###
-  # preprocess: possibly change the given SalsaTigerSentence 
+  # preprocess: possibly change the given SalsaTigerSentence
   # to enable better learning
   def preprocess(sent)           # SalsaTigerSentence object
 
-
-    if @dataset == "train" and 
+    # @todo AB: [2015-12-16 Wed 15:39]
+    #   Don't think it should be done by Rosy, do it only in Frappe.
+    #   This module will be moved to Frappe's lib.
+    if @dataset == "train" and
         (@exp.get("fe_syn_repair") or @exp.get("fe_rel_repair"))
       FixSynSemMapping.fixit(sent, @exp, @interpreter_class)
     end
@@ -226,8 +228,8 @@ class InputData
                           frame) # FrameNode
 
     # target POS
-    if frame.target()
-      main_target = @interpreter_class.main_node_of_expr(frame.target.children(), "no_mwe")
+    if frame.target
+      main_target = @interpreter_class.main_node_of_expr(frame.target.children, "no_mwe")
     else
       main_target = nil
     end
@@ -236,8 +238,8 @@ class InputData
     else
       target_pos = nil
     end
-    if frame.target()
-      target_str = frame.target().yield_nodes_ordered().map { |t_node| 
+    if frame.target
+      target_str = frame.target.yield_nodes_ordered.map { |t_node|
         if t_node.is_syntactic?
           @interpreter_class.lemma_backoff(t_node)
         else
@@ -248,9 +250,9 @@ class InputData
     else
       target_str = ""
     end
-            
-    @failed_parses.register(construct_instance_id(sent.id(), frame.id()),
-                            frame.name(),
+
+    @failed_parses.register(Rosy::construct_instance_id(sent.id, frame.id),
+                            frame.name,
                             target_str,
                             target_pos,
                             frame.children.map { |fe| fe.name })
@@ -263,28 +265,28 @@ class InputData
                            feature_value, # object
                            extractor_obj) # AbstractFeatureExtractor object
 
-    if extractor_obj.class.sql_type() =~ /(\d+)/
+    if extractor_obj.class.sql_type =~ /(\d+)/
       # sql type contains some statement about the length.
       # just crudely compare to feature length
       length = $1.to_i
       if feature_value.class == String and
-          feature_value.length() > length
+          feature_value.length > length
 
         if feature_name == "sentid"
-	  print length;
+          print length;
           print feature_value;
-	  print feature_value.length();
-	  # if the sentence (instance) ID is too long, we cannot go on.
+          print feature_value.length;
+          # if the sentence (instance) ID is too long, we cannot go on.
           $stderr.puts "Error: Instance ID is longer than its DB column."
           $stderr.puts "Please increase the DB column size in {Tiger,Collins}FeatureExtractors.rb"
           raise "SQL entry length surpassed"
 
         elsif @exp.get("verbose")
-          # KE Feb 07: don't print warning, 
+          # KE Feb 07: don't print warning,
           # this is just too frequent
           # for other features, we just issue a warning, and only if we are verbose
 
-          # $stderr.puts "Warning: feature #{feature_name} longer than its DB column (#{length.to_s} vs #{feature_value.length()}): #{feature_value}"                      
+          # $stderr.puts "Warning: feature #{feature_name} longer than its DB column (#{length.to_s} vs #{feature_value.length}): #{feature_value}"
         end # feature name check
       end # length surpassed
     end # length found in sql type
@@ -292,3 +294,5 @@ class InputData
   end
 
 end
+end
+end