shalmaneser 0.0.1.alpha → 1.2.0.rc1

data/lib/common/Maxent.rb

@@ -11,21 +11,32 @@ class Maxent
   ###
   def initialize(program_path,parameters)
     
-    if parameters.empty?	
-      puts "Error: The OpenNLP maxent system needs two paths (first the location of maxent itself and then the location of the interface, usually program/tools/maxent)."
-      puts "I got only the program path."
-      Kernel.exit
-    end
-
+    # @note AB: <parameters> is an Array with the last part of the
+    #   line from the experiment file, it should contain the path to our
+    #   java wrappers, but we don't want it.
+    #   Since the presence of this part is checked only here we
+    #   suppose it obsolete and set this path manually here.
+    # if parameters.empty?	
+    #   puts "Error: The OpenNLP maxent system needs two paths (first the location of maxent itself and then the location of the interface, usually program/tools/maxent)."
+    #   puts "I got only the program path."
+    #   Kernel.exit
+    # end
+    # @interface_path = parameters.first
+
+    # @note AB: Setting path manually.
+    #   It assumes <Maxent.rb> ist in <lib/common> and
+    #   <Classify.class> is in <lib/ext/maxent>.
+    @interface_path = File.expand_path('../ext/maxent', File.dirname(__FILE__))
+      
     @maxentpath = program_path
-    @interface_path = parameters.first
+
     unless @maxentpath =~ /\/$/
       @maxentpath = @maxentpath + "/"
     end
     
     # classpath for maxent
     
-    @cp = "#{ENV["CLASSPATH"]}:#{@maxentpath}:#{@maxentpath}lib:#{@maxentpath}lib/trove.jar:#{@maxentpath}output/maxent-2.4.0.jar"
+    @cp = "#{@maxentpath}:#{@maxentpath}lib:#{@maxentpath}lib/trove.jar:#{@maxentpath}output/maxent-2.4.0.jar:#{ENV["CLASSPATH"]}"
 
   end
 
@@ -148,9 +159,9 @@ class Maxent
       return nil
     end
 
-    retv = Array.new()
+    retv = []
 
-    f.each { |line|
+    f.each do |line|
       line_results = Array.new()
       pieces = line.split() # split at whitespace
 
@@ -164,8 +175,11 @@ class Maxent
 
       # sort: most confident label first
       retv << line_results.sort {|a,b| b[1] <=> a[1]}
-    }
-    return retv
+    end
+
+    f.close
+
+    retv
   end
 
   

data/lib/common/Parser.rb

@@ -72,12 +72,12 @@ end
 
 
 class FilePartsParser
-  # @file = File object for the corpus
-  # @head = string up to the first <s> tag
-  # @tail = string after the last </s> tag
-  # @rest = string starting with the latest <s> tag (complete this to
+  # <@file> = File object for the corpus
+  # <@head> = string up to the first <s> tag
+  # <@tail> = string after the last </s> tag
+  # <@rest> = string starting with the latest <s> tag (complete this to
   # a <s>...</s> structure by reading up to next </s> tag)
-  # @readCompletely = boolean specifying whether there's still something
+  # <@readCompletely> = boolean specifying whether there's still something
   # left to read in the file
 
   attr_reader :head, :tail

data/lib/common/SynInterfaces.rb

@@ -245,12 +245,19 @@ class SynInterfaces
 end
 
 
-require "common/CollinsInterface"
-require "common/BerkeleyInterface"
-require "common/SleepyInterface"
-require "common/MiniparInterface"
-require "common/TntInterface"
-require "common/TreetaggerInterface"
+# AB: TODO We should require programmatically all files in
+# <frprep/interpreters> and <frprep/interfaces>.
+require "frprep/CollinsInterface"
+
+require 'frprep/interfaces/berkeley_interface'
+require 'frprep/interpreters/berkeley_interpreter'
+require 'frprep/interfaces/stanford_interface'
+require 'frprep/interpreters/stanford_interpreter'
+
+require "frprep/SleepyInterface"
+require "frprep/MiniparInterface"
+require "frprep/TntInterface"
+require "frprep/TreetaggerInterface"
 
 
 class EmptyInterpreter < SynInterpreter

data/lib/common/TabFormat.rb

@@ -25,15 +25,16 @@ require "tempfile"
 require "common/ISO-8859-1"
 require "common/ruby_class_extensions"
 
+# @todo Remove this definition in the top namespace!
 #######################
 # This function takes a variable number of arguments and
 # returns them as an array
 # Idea: make formulation of tab format entries easier to read,
 # enclose variable arguments in a repeat() call,
 # which immediately gets transformed into a list
-def repeat(*args)
-  return args
-end
+#def repeat(*args)
+#  return args
+#end
 
 #######################
 class TabFormatFile
@@ -84,7 +85,7 @@ class TabFormatFile
   # (in terms of number of lines) as the first file.
   # each sentence is returned in the form of an
   # array of TabFormatSentence sentences.
-  
+  # AB: TODO Delete this nasty exception!!!
   def each_sentence
     unless @read_completely
       sentence = @my_sentence_class.new(@patterns)
@@ -98,9 +99,9 @@ class TabFormatFile
 	  }	
 	#STDERR.puts linearray
 	  @no_of_read_lines += 1
-	  if linearray.detect{|x| x.strip == ""}
+	  if linearray.detect {|x| x.strip == ""}
 	    if linearray.detect {|x| x.strip != ""}
-	      STDERR.puts "Error: Mismatching empty lines!" 
+	      STDERR.puts "Error: Mismatching empty lines! <from lib/common>" 
 	      exit(1)
 	    else
 	      # sentence finished. yield it and start a new one

data/lib/common/Tiger.rb

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
-require "common/headz"
-require "common/SalsaTigerRegXML"
-require "common/ruby_class_extensions"
+require 'common/headz'
+require 'common/SalsaTigerRegXML'
+require 'common/ruby_class_extensions'
 class Array
   include EnumerableDistribute
 end
 
 
-require "common/AbstractSynInterface"
+require 'common/AbstractSynInterface'
 
 #############################################
 #

data/lib/common/Timbl.rb

@@ -0,0 +1,144 @@
+# wrapper script for timbl learner
+# sp 24 08 04
+
+# contract for Learner classes:
+
+
+class Timbl
+  
+  def initialize(program_path, parameters)
+    
+#    @timblpath="/proj/llx/Software/MachineLearning/Timbl5/Timbl "
+
+    @timblpath = File.join(program_path,"Timbl")
+    unless @timblpath =~ /\s$/
+      # path must end in space so we can just attach parameters
+      @timblpath << " "
+    end
+
+    if parameters.empty?
+      # was: +vs
+      @params = "-mM -k5 +vs" # default parameters
+    else
+      @params = parameters.join(" ") + " "
+    end
+  end
+
+  def timbl_out_to_malouf_out(infilename,outfilename) # timbl: [all features], [gold standard label]
+    infile = File.new(infilename)
+    outfile = File.new(outfilename,"w")
+    while (line = infile.gets)
+      larray = line.chomp.split(",")
+      ml_label = larray.last
+      outfile.puts ml_label+"\t1"
+    end
+    infile.close
+    outfile.close
+  end
+
+  def train(infile,classifier_location)                  # lazy learning: for training, store the 
+                                                         # instancebase as a tree (TiMBL -I / -i option)
+    # figure out how many features we have
+    f = File.new(infile)
+    line = f.gets().chomp()
+    num_features = line.split(",").length() - 1
+
+    # and train
+    if classifier_location then
+      @instancebase = classifier_location
+    else
+      @instancebase = infile+".instancebase"
+    end
+    successfully_run(@timblpath+@params+" -N#{num_features} -f "+infile+" -I "+@instancebase)
+  end
+
+  # return true iff reading the classifier has had success
+  def read(classifierfile)
+    unless FileTest.exists?(classifierfile)
+      STDERR.puts "[Timbl] Cannot find instancebase at #{classifierfile}"
+      return false
+    end
+    @instancebase = classifierfile
+    return true
+  end
+
+  def exists?(classifierfile)
+    return FileTest.exists?(classifierfile)
+  end
+
+  def write(classifierfile)
+    %x{cp #{@instancebase} #{classifierfile}} # store training data as "modelfile"
+    File.chmod(0664,classifierfile)
+  end
+  
+  def apply(infile,outfile)
+    temp_outfile = outfile+".temp"
+    successfully_run(@timblpath+@params+" -i "+@instancebase+" -t "+infile+" -o "+temp_outfile)    
+    
+    # if we have an empty input file, timbl will not produce an output file
+    unless FileTest.exists?(temp_outfile)      
+#      STDERR.puts "[Timbl] Warning: Timbl failed to produce an outfile."
+      return false
+    end
+    
+    # no error
+    timbl_out_to_malouf_out(temp_outfile,outfile)
+    File.unlink(temp_outfile)
+
+    # true iff outfile exists
+    if  FileTest.exists?(outfile)
+      return true
+    else
+#      STDERR.puts "[Timbl] Warning: Final outfile could not be produced."
+      return false
+    end
+      
+  end
+
+  #####
+  def read_resultfile(filename)
+    begin
+      f = File.new(filename)
+    rescue
+      $stderr.puts "TiMBL error: cannot read TiMBL result file #{filemame}."
+      return nil
+    end
+
+    retv = Array.new()
+
+    f.each { |line|
+      line_results = Array.new()
+      pieces = line.split()
+
+      while not(pieces.empty?)
+        label = pieces.shift()
+
+        begin
+          confidence = pieces.shift().to_f()
+        rescue
+          $stderr.puts "Error reading mallet output: invalid line: #{line}"
+          confidence = 0
+        end
+
+        line_results << [label, confidence]
+      end
+      retv << line_results
+    }
+
+    return retv
+  end
+  
+  #########################
+  private
+
+  ###
+  def successfully_run(command)
+    retv = Kernel.system(command)
+    unless retv
+      $stderr.puts "Error running classifier. Exiting."
+      $stderr.puts "Offending command: "+command
+      exit 1
+    end
+  end
+
+end

data/lib/common/{FrprepHelper.rb → frprep_helper.rb}

@@ -10,7 +10,6 @@ require "common/AbstractSynInterface"
 
 ############################################3
 # Module FrprepHelper:
-#
 # diverse transformation methods for frprep.rb
 # moved over here to make the main file less crowded
 module FrprepHelper
@@ -162,23 +161,28 @@ module FrprepHelper
       raise "Could not read #{input_filename}, or could not write to #{output_filename}."
     end
 
-    filename_core = File.basename(input_filename, "txt")
+    # AB: TODO This assumes all input files have the extension <txt>.
+    # Is it good?
+    filename_core = File.basename(input_filename, 'txt')
 
     # array(string): keep the words of each sentence
-    sentence = Array.new
+    sentence = []
     # sentence number for making the sentence ID:
     # global count, over all input files
     sentno = 0
     
-    while (line = infile.gets())
+    while line = infile.gets
       
       # make a sentence ID for the next sentence: running number
-      sentid = filename_core + "_" + sentno.to_s
+      sentid = "#{filename_core}_#{sentno}"
       sentno += 1
       
       # read words into the sentence array,
       # separating out punctuation attached to the beginning or end of words
-      sentence.clear()
+      sentence.clear
+
+      # AB: TODO Remove this naive tokenizer, better to have a fully
+      # tokenized input using an external tokenizer than that.
       line.split.each { |word|
         # punctuation at the beginning of the word
         #if word =~ /^([\(\[`'\"-]+)(.*)$/
@@ -206,7 +210,9 @@ module FrprepHelper
       }
 
 
+      
       # remove empty words
+      # AB: TODO Is it possible? Remove this.
       sentence.reject! { |word| word.nil? or word.strip.empty? }
 
       # write words to tab file
@@ -222,13 +228,13 @@ module FrprepHelper
       }
       outfile.puts
     end
-    outfile.close()
+    outfile.close
   end
 
   ###########
   # 
   # class method split_dir: 
-  # read all files in one directory and produce chunk files *#{suffix} in outdir
+  # read all files in one directory and produce chunk files with _suffix_ in outdir
   # with a certain number of files in them (sent_num).
   # Optionally, remove all sentences longer than sent_leng
   #

data/lib/common/headz.rb

@@ -27,7 +27,7 @@
 #   print "preposition of conjunction involved"
 # end
 
-require "common/SalsaTigerRegXML"
+require 'common/SalsaTigerRegXML'
 
 class Headz 
 

data/lib/common/ruby_class_extensions.rb

@@ -234,7 +234,7 @@ end
 ################
 module EnumerableBool
   ###
-  # And_{x \in X} block(x)
+  # And_(x \in X) block(x)
   def big_and(&block)
     each { |x|
       unless block.call(x)
@@ -245,7 +245,7 @@ module EnumerableBool
   end
 
   ###
-  # Or_{x \in X} block(x)
+  # Or_(x \in X) block(x)
   def big_or(&block)
     each { |x|
       if block.call(x)
@@ -256,7 +256,7 @@ module EnumerableBool
   end
 
   ###
-  # Sum_{x \in X} block(x)
+  # Sum_(x \in X) block(x)
   def big_sum(init = 0, &block)
     sum = init
     unless block_given?

data/lib/fred/FredBOWContext.rb

@@ -432,7 +432,16 @@ class NoncontiguousContextProvider < AbstractContextProvider
       File.symlink(filename, frprep_in + "infile")
 
       # call frprep
-      retv = Kernel.system("ruby frprep.rb -e #{tf_exp_frprep.path()}")
+      # AB: Bad hack, find a way to invoke FrPrep directly.
+      # We will need an FrPrep instance and an options object.
+      base_dir_path = File.expand_path(File.dirname(__FILE__) + '/../..')
+
+      # Remove this
+      FileUtils.cp(tf_exp_frprep.path, '/tmp/frprep.exp')
+      # after debugging
+
+      retv = system("ruby -rubygems -I #{base_dir_path}/lib #{base_dir_path}/bin/frprep -e #{tf_exp_frprep.path}")
+
       unless retv
         $stderr.puts "Error analyzing #{filename}. Exiting."
         exit 1
@@ -469,6 +478,8 @@ class NoncontiguousContextProvider < AbstractContextProvider
     
     # remove temporary data
     temptable_obj.drop_temp_table()
+
+    # AB: TODO Rewrite this passage using pure Ruby.
     %x{rm -rf #{frprep_in}}
     %x{rm -rf #{frprep_out}}
     %x{rm -rf #{frprep_dir}}
@@ -487,6 +498,7 @@ class NoncontiguousContextProvider < AbstractContextProvider
   # - hash table containing all hash keys
   def make_index(dir)
 
+    # AB: Why this limits? Use constants!
     space_for_sentstring = 30000
     space_for_hashkey = 500
 
@@ -644,7 +656,7 @@ class NoncontiguousContextProvider < AbstractContextProvider
       end
       # and enter recursion
       remove_files(subdir)
-      File.rm_f(subdir)
+      FileUtils.rm_f(subdir)
     }
   end