shalmaneser 1.2.0.rc4 → 1.2.rc5

data/lib/common/ISO-8859-1.rb

@@ -1,24 +0,0 @@
-# KE changed July 05: now no inclusion of modules required,
-# and names changed from REXML.Encodign to UtfIso
-
-module UtfIso
-  # Convert from UTF-8
-  def UtfIso.to_iso_8859_1(content)
-    array_utf8 = content.unpack('U*')
-    array_enc = []
-    array_utf8.each do |num|
-      if num <= 0xFF
-        array_enc << num
-      else
-        # Numeric entity (&#nnnn;); shard by  Stefan Scholl
-        #	   array_enc += to_iso_8859("&\##{num};").unpack('C*')
-      end
-    end
-    array_enc.pack('C*')
-  end
-
-  # Convert to UTF-8
-  def UtfIso.from_iso_8859_1(str)
-    str.unpack('C*').pack('U*')
-  end
-end

data/lib/common/ML.rb

@@ -1,186 +0,0 @@
-# sp 24 08 04
-
-# this file provides a very simple wrapper for using different ML systems
-# all you need to do is to write the appropriate learner class
-# and insert them in the initialize routine here in ML()
-#
-# available at the moment: 
-# * timbl (memory-based learner) 
-# * mallet-maxent (another maxent system)
-# * maxent (the OpenNLP maxent system)
-
-# part of contract: learner is not initialised unless it is either trained or read
-
-require "common/Optimise"
-
-class Classifier
-
-  @@learners = [
-    ["timbl", "Timbl", "Timbl"],
-#    ["mallet", "Mallet", "Mallet"],
-    ["maxent", "Maxent", "Maxent"]
-  ]
-
-  def initialize(learner,params)
-
-    @ready = false
-
-    if params[0] == "optimise"
-      params.shift
-      @optimise = true
-    else
-      @optimise = false      
-    end
-
-    program_path = ""
-    begin
-      program_path = params.shift.chomp
-      unless FileTest.exist? program_path
-        $stderr.puts "Error: Could not find classifier system at " + program_path
-        $stderr.puts "Perhaps an erroneous entry in your experiment file?"
-        exit 1
-      end
-    rescue NoMethodError
-      $stderr.puts "Error: No program path provided for classifier system."
-    end
-
-    # try to find our learner in the pre-set list of learners
-    learner_tuple = @@learners.assoc(learner)
-    unless learner_tuple
-      $stderr.puts "Error: I don't know the learner " + learner.to_s
-      $stderr.puts "Perhaps an erroneous entry in your experiment file?"
-      exit 1
-    end
-
-    learner_name, learner_filename, learner_classname = learner_tuple
-    require "common/#{learner_filename}"
-    @learner = eval(learner_classname).new(program_path,params)
-  end
-
-  # a classifier can (and has to be) either trained or read
-  def train(trainfile, classifier_file=nil)    
-    # train on the training data in trainfile
-    # make sure we produce a valid file name
-    
-    # it is possible to directly specify a filename for storing the classifier
-
-    trainfile.gsub!(/[<>]/,"")
-    trainfile.gsub!(/ /,"_")
-    if @optimise
-      STDERR.puts "[ML] using feature optimisation"
-      @optimiser = Optimise.new
-      @optimiser.init_from_data(trainfile)
-      optimisedfile = trainfile+".opted"
-      @optimiser.apply(trainfile,optimisedfile)
-      @learner.train(optimisedfile,classifier_file)
-      File.delete(optimisedfile)
-    else
-      STDERR.puts "[ML] no feature optimisation"
-      @learner.train(trainfile,classifier_file)      
-    end
-    @ready = true
-  end
-
-  
-  # returns true iff reading the classifier from the file has had success
-  
-  def read(classifier_file)
-    # make sure we produce a valid file name
-    classifier_file.gsub!(/[<>]/,"")
-    classifier_file.gsub!(/ /,"_")
-    
-    # read file, if present    
-    
-    status = @learner.read(classifier_file)
-    
-    # if reading has failed, return "false"
-    unless status
-      STDERR.puts "reading from #{classifier_file} did not succeed"
-      return status
-    end
-    
-    # read optimisation, if desired
-    if @optimise
-      optimisations_filename = Optimise.recommended_filename(classifier_file)
-      unless FileTest.exists? optimisations_filename
-        STDERR.puts "[ML] Error: attempted to read stored optimisation, but file does not exist"
-        return false
-      else
-	@optimiser = Optimise.new
-        @optimiser.init_from_file(optimisations_filename)
-      end
-    end
-    
-    @ready = true
-    return true
-    
-  end
-
-  # a classifier can be stored somewhere. This can be more than one file (classifier-specific), 
-  # but all files start with "classifier_file"
-
-  def write(classifier_file)
-    # make sure we produce a valid file name
-    classifier_file.gsub!(/[<>]/,"")
-    classifier_file.gsub!(/ /,"_")
-    @learner.write(classifier_file)
-    if @optimise
-      @optimiser.store(Optimise.recommended_filename(classifier_file))
-    end
-  end
-
-  ###
-  # exists?
-  # check if a classifier is living at some particular path
-
-  def exists?(classifier_file)
-    classifier_file.gsub!(/[<>]/,"")
-    classifier_file.gsub!(/ /,"_")
-    return @learner.exists?(classifier_file)
-  end
-  
-  # a classifier can be applied 
-
-  # returns true iff application has had success
-  
-  def apply(testfile,outfile) # test either on the training or the test data in the specified dir
-    # make sure we produce a valid file name
-    testfile.gsub!(/[<>]/,"")
-    testfile.gsub!(/ /,"_")
-    # make sure we produce a valid file name
-    outfile.gsub!(/[<>]/,"")
-    outfile.gsub!(/ /,"_")
-    
-    unless @ready
-      STDERR.puts "[ML] Warning: learner not ready for testing! Must be trained or read."
-      return false
-    end    
-    
-    # do we have a testfile?
-    
-    unless FileTest.exists?(testfile)
-      STDERR.puts "[ML] Warning: could not find testfile (maybe empty test set?)."
-      return false
-    end
-    
-    if @optimise      
-      optimisedfile = testfile+".opted"
-      @optimiser.apply(testfile,optimisedfile)
-      return @learner.apply(optimisedfile,outfile)
-      File.delete(optimisedfile)
-    else
-      return @learner.apply(testfile,outfile)
-    end
-    
-  end
-
-  ###
-  # read classifier result file,
-  # returns a list of instance_results
-  # where an instance_result is a list of pairs [label, confidence]
-  # where the pairs are sorted by confidence
-  def read_resultfile(file)
-    return @learner.read_resultfile(file)
-  end
-
-end

data/lib/common/Mallet.rb

@@ -1,236 +0,0 @@
-# wrapper script for the Mallet toolkit Maxent classifier
-
-# Problem with Winnow: cannot be serialised (written to file). Support dropped.
-
-# sp 27 10 04
-
-
-require "tempfile"
-require "ftools"
-
-class Mallet
-
-  ###
-  def initialize(program_path,parameters)
-    
-    if parameters.empty?	
-      puts "Error: Mallet needs two paths (first the location of mallet itself and then the location of the interface, usually program/tools/mallet)."
-      puts "I got only the program path."
-      Kernel.exit
-    end
-
-    @malletpath = program_path
-    @interface_path = parameters.first
-    unless @malletpath =~ /\/$/
-      @malletpath = @malletpath + "/"
-    end
-    
-    @learner = "MaxEnt,gaussianPriorVariance=1.0"
-
-    # classpath for mallet
-    
-    @cp = "#{ENV["CLASSPATH"]}:#{@malletpath}class:#{@malletpath}lib/bsh.jar"
-
-  end
-
-  ###
-  def train(infilename,classifier_location)
-    csvfile = Tempfile.new(File.basename(infilename)+".csvtrain")
-    infile = File.new(infilename)
-    c45_to_csv(infile,csvfile) # training data in csv format
-    infile.close
-    csvfile.close
-    @mallet_train_vectors = infilename+".trainvectors" # training data in mallet format
-    if classifier_location
-      @classifier_mallet_path = classifier_location
-    else
-      @classifier_mallet_path = infilename+".classifier"
-    end
-
-    command1 = [@malletpath+"bin/csv2vectors ",
-		    " --input ",csvfile.path,
-		    " --output ",@mallet_train_vectors].join("")
-
-    command2 = ["cd #{@interface_path}; ",
-                "java -cp #{@cp} -Xmx1000m Train ",
-                " --train ",@mallet_train_vectors,
-                " --out ",@classifier_mallet_path,
-                " --trainer ",@learner].join("")
-#    STDERR.puts "[train 1] "+command1
-    successfully_run(command1) # encode
-#    STDERR.puts "[train 2] "+command2
-    successfully_run(command2) # train
-    csvfile.close(true)
-  end
-
-  def write(classifier_file)
-    if @classifier_mallet_path
-      %x{cp #{@classifier_mallet_path} #{classifier_file}.classifier} # store classifier
-   #    File.chmod(0664,classifier_file+".classifier")
-    end
-    if @mallet_train_vectors
-      %x{cp #{@mallet_train_vectors} #{classifier_file}.trainvectors} # store train vectors to recreate pipe for testing data
-#      File.chmod(0664,classifier_file+".trainvectors")
-    end
-  end
-
-  ###
-  def exists?(classifier_file)
-    return (FileTest.exists?(classifier_file+".trainvectors") and
-              FileTest.exists?(classifier_file+".classifier"))
-  end
-
-  ###
-  # return true iff reading the classifier has had success
-  def read(classifier_file)
-    @mallet_train_vectors = classifier_file+".trainvectors" # training data in mallet format
-    @classifier_mallet_path = classifier_file+".classifier"
-    unless FileTest.exists?(@mallet_train_vectors)
-      $stderr.puts "No classifier file "+@mallet_train_vectors
-      return false
-    end
-    unless FileTest.exists?(@classifier_mallet_path)
-      $stderr.puts "No classifier file "+@classifier_mallet_path
-      return false
-    end
-    return true
-  end  
-  
-  ###
-  def apply(infilename,outfilename)
-    unless @classifier_mallet_path and @mallet_train_vectors
-      return false
-    end
-
-    #    STDERR.puts "Testing on "+infilename
-    csvfile = Tempfile.new(File.basename(infilename)+".csvtest")
-
-    infile = File.new(infilename)
-    c45_to_csv(infile,csvfile) # training data in csv format
-    infile.close
-    csvfile.close
-
-    test_mallet_path = infilename+".test.vectors" # training data in mallet format
-
-    # $stderr.puts "test file in " + infilename
-    # $stderr.puts "using training vectors from " + @mallet_train_vectors
-
-    # copy train vectors to temp file.
-    # reason: mallet in std edition reads _and writes_ this file
-    # if rosy is interrupted, corrupted (ie incomplete) train vector files
-    # result
-
-    tempfile = Tempfile.new("mallet")
-    tempfilename = tempfile.path
-    unless File.copy(@mallet_train_vectors,tempfilename)    
-      return false
-    end
-
-    command1 = [@malletpath+"bin/csv2vectors", # encode testing data
-                " --input ",csvfile.path,
-                " --output ",test_mallet_path,
-                " --use-pipe-from ",tempfilename].join("")
-
-#    $stderr.puts "Mallet encode: " + command1
-    unless successfully_run(command1) # encode
-      return false
-    end
-
-    File.safe_unlink(tempfilename)
-    
-    # some error in encoding?
-    unless FileTest.exists?(test_mallet_path)
-      return false
-    end
-
-    command2 = ["cd #{@interface_path}; ",
-                "java -cp #{@cp} -Xmx1000m Classify ",
-                @classifier_mallet_path," ",
-                test_mallet_path," ",
-                "> ",outfilename].join("")
-    
-    # classify
-#    $stderr.puts "Mallet classify: " + command2
-    unless    successfully_run(command2)
-      return false
-    end
-
-    # some error in classification
-    unless FileTest.exists?(outfilename)
-      return false
-    end
-
-     # no errors = success
-    csvfile.close(true)
-    return true
-  end
-
-  #####
-  # format of Mallet result file:
-  # <best label> <confidence> \t <secondbest_label> <confidence>....
-  def read_resultfile(filename)
-    begin
-      f = File.new(filename)
-    rescue
-      $stderr.puts "Mallet error: cannot read Mallet result file #{filemame}."
-      return nil
-    end
-
-    retv = Array.new()
-
-    f.each { |line|
-      line_results = Array.new()
-      pieces = line.split()
-
-      while not(pieces.empty?)
-        label = pieces.shift()
-
-        begin
-          confidence = pieces.shift().to_f()
-        rescue
-          $stderr.puts "Error reading mallet output: invalid line: #{line}"
-          confidence = 0
-        end
-
-        line_results << [label, confidence]
-      end
-      retv << line_results
-    }
-
-    return retv
-  end
-
-  
-  ###################################
-  private
-
-  ###
-  # mallet needs "comma separated values"-file
-  # input: features separated by comma
-  # output:
-  # line_number classlabel features_joined_by_spaces
-  def c45_to_csv(inpipe,outpipe) 
-    idx = 0
-    while (line = inpipe.gets)
-      line.chomp!
-      idx += 1
-      la = line.split(",")
-      label = la.pop
-      if label[-1,1] == "."
-	label.chop!
-      end
-      outpipe.puts [idx,label].join(" ")+" "+la.join(" ")
-    end
-  end
-
-  ###
-  def successfully_run(command)
-    retv = Kernel.system(command)
-    unless retv
-      $stderr.puts "Error running classifier. Continuing."
-      $stderr.puts "Offending command: "+command
- #     exit 1
-    end
-    return retv
-  end
-end

data/lib/common/Maxent.rb

@@ -1,229 +0,0 @@
-# wrapper script for the OpenNLP Maxent classifier
-
-# sp July 2007
-
-
-require "tempfile"
-require 'fileutils'
-
-class Maxent
-
-  ###
-  def initialize(program_path,parameters)
-    
-    # @note AB: <parameters> is an Array with the last part of the
-    #   line from the experiment file, it should contain the path to our
-    #   java wrappers, but we don't want it.
-    #   Since the presence of this part is checked only here we
-    #   suppose it obsolete and set this path manually here.
-    # if parameters.empty?	
-    #   puts "Error: The OpenNLP maxent system needs two paths (first the location of maxent itself and then the location of the interface, usually program/tools/maxent)."
-    #   puts "I got only the program path."
-    #   Kernel.exit
-    # end
-    # @interface_path = parameters.first
-
-    # @note AB: Setting path manually.
-    #   It assumes <Maxent.rb> ist in <lib/common> and
-    #   <Classify.class> is in <lib/ext/maxent>.
-    @interface_path = File.expand_path('../ext/maxent', File.dirname(__FILE__))
-      
-    @maxentpath = program_path
-
-    unless @maxentpath =~ /\/$/
-      @maxentpath = @maxentpath + "/"
-    end
-    
-    # classpath for maxent
-    
-    @cp = "#{@maxentpath}:#{@maxentpath}lib:#{@maxentpath}lib/trove.jar:#{@maxentpath}output/maxent-2.4.0.jar:#{ENV["CLASSPATH"]}"
-
-  end
-
-  ###
-  #
-  # write classifier to training directory...
-  def train(infilename,classifier_file)
-    trainfile = Tempfile.new(File.basename(infilename)+".maxenttrain")
-    infile = File.new(infilename)
-    c45_to_maxent(infile,trainfile) # training data in csv format
-    infile.close
-    trainfile.close
-
-    if classifier_file
-      @classifier_location = classifier_file
-    else
-      @classifier_location = trainfile.path+"Model.bin.gz"
-    end
-    
-    @classifier_location = enforce_compact_storage(@classifier_location)
-
-    # store model in binary, gzipped form...
-    command = ["cd #{@interface_path}; ",
-                #"/usr/lib/jvm/java-1.7.0/bin/java -cp #{@cp} -Xmx1000m Train",
-		"java -cp #{@cp} -Xmx1000m Train",
-               trainfile.path,
-               @classifier_location].join(" ")
-    # remember location
-    unless  successfully_run(command)
-      return false
-    end
-    trainfile.close(true)
-  end
-
-  def write(classifier_file)
-    
-    classifier_file = enforce_compact_storage(classifier_file)
-    
-    if @classifier_location
-      @classifier_location = enforce_compact_storage(@classifier_location)
-      %x{cp #{@classifier_location} #{classifier_file}} # store classifier
-   #    File.chmod(0664,classifier_file+".classifier")
-    else
-      $stderr.puts "Maxent error: cannot read Maxent classifier file #{@classifier_file}."
-      return nil      
-    end
-  end
-
-  ###
-  def exists?(classifier_file)
-    classifier_file = enforce_compact_storage(classifier_file)    
-    return FileTest.exists?(classifier_file)
-  end
-  
-  ###
-  # return true iff reading the classifier has had success
-  def read(classifier_file)
-    
-    classifier_file = enforce_compact_storage(classifier_file)
-
-    if exists?(classifier_file)
-      @classifier_location = classifier_file
-      return true
-    else
-      $stderr.puts "No classifier file "+classifier_file
-      return false
-    end
-  end
-  
-  ###
-  def apply(infilename,outfilename)
-    
-    @classifier_location = enforce_compact_storage(@classifier_location)
-    unless @classifier_location
-      return false
-    end
-
-    testfile = Tempfile.new(File.basename(infilename)+".maxenttrain")
-    
-    infile = File.new(infilename)
-    c45_to_maxent(infile,testfile) # training data in csv format
-    infile.close
-    testfile.close
-    
-    command = ["cd #{@interface_path}; ",
-               #"/usr/lib/jvm/java-1.7.0/bin/java -cp #{@cp} -Xmx1000m Classify ",
-               "java -cp #{@cp} -Xmx1000m Classify ",
-               testfile.path,
-               @classifier_location,
-               ">",
-               outfilename].join(" ")
-    
-    # classify
-    unless  successfully_run(command)
-      return false
-    end
-    
-    # some error in classification
-    unless FileTest.exists?(outfilename)
-      return false
-    end
-    
-    # no errors = success
-    testfile.close(true)
-    return true
-  end
-
-  #####
-  # format of Maxent result file:
-  # <best label>[<confidence>]  <secondbest_label>[<confidence>] ....
-  #
-  # returns a list of instance_results
-  # where an instance_result is a list of pairs [label, confidence]
-  # where the pairs are sorted by confidence
-  def read_resultfile(filename)
-    begin
-      f = File.new(filename)
-    rescue
-      $stderr.puts "Maxent error: cannot read Maxent result file #{filemame}."
-      return nil
-    end
-
-    retv = []
-
-    f.each do |line|
-      line_results = Array.new()
-      pieces = line.split() # split at whitespace
-
-      pieces.each {|piece|
-        piece =~ /(\S+)\[(.+)\]/
-        label = $1
-        confidence = $2.to_f
-        
-        line_results << [label, confidence]        
-      }
-
-      # sort: most confident label first
-      retv << line_results.sort {|a,b| b[1] <=> a[1]}
-    end
-
-    f.close
-
-    retv
-  end
-
-  
-  ###################################
-  private
-
-  ###
-  # produce input file for maxent learner: make attribute-value pairs
-  # where attribute ==    featureX=
-  def c45_to_maxent(inpipe,outpipe) 
-    while (line = inpipe.gets)
-      line.chomp!
-      la = line.split(",")
-      label = la.pop
-      if label[-1,1] == "."
-	label.chop!
-      end
-      la.each_index {|i|
-        la[i] = i.to_s() + "=" + la[i]
-      }
-      la.push(label)
-      outpipe.puts la.join(" ")
-    end
-  end
-
-  # since the OpenNLP MaxEnt system determines storage based on filename,
-  # make sure that all models are stored internally as binary, gzipped files.
-  
-  def enforce_compact_storage(filename)
-    if filename =~ /Model.bin.gz/
-      return filename
-    else
-      return filename+"Model.bin.gz"
-    end
-  end
-
-  ###
-  def successfully_run(command)
-    retv = Kernel.system(command)
-    unless retv
-      $stderr.puts "Error running classifier. Continuing."
-      $stderr.puts "Offending command: "+command
- #     exit 1
-    end
-    return retv
-  end
-end