shalmaneser-lib 1.2.rc5

data/lib/logging.rb

@@ -0,0 +1,25 @@
+require 'logger'
+require 'pastel'
+
+# A general logger for all instances.
+module Shalmaneser
+  # @todo AB: [2015-12-31 Thu 13:15]
+  #   Check if the constant is defined.
+  #   Do not rely only on the require order.
+  LOGGER = Logger.new($stderr)
+
+  LOGGER.level = Logger.const_get(ENV.fetch('LOG_LEVEL', 'INFO'))
+  pastel = Pastel.new
+  colors = {
+    'FATAL' => pastel.red.bold.detach,
+    'ERROR' => pastel.red.detach,
+    'WARN'  => pastel.yellow.detach,
+    'INFO'  => pastel.green.detach,
+    'DEBUG' => pastel.white.detach
+  }
+
+  LOGGER.formatter = lambda do |severity, datetime, progname, message|
+    colorizer = $stderr.tty? ? colors[severity] : ->(s) { s }
+    "#{colorizer.call(severity)}: #{message}\n"
+  end
+end

data/lib/ml/classifier.rb

@@ -0,0 +1,189 @@
+# sp 24 08 04
+
+# this file provides a very simple wrapper for using different ML systems
+# all you need to do is to write the appropriate learner class
+# and insert them in the initialize routine here in ML()
+#
+# available at the moment:
+# * timbl (memory-based learner)
+# * mallet-maxent (another maxent system)
+# * maxent (the OpenNLP maxent system)
+
+# part of contract: learner is not initialised unless it is either trained or read
+
+# @note AB: This is only a remark about dynamic requirement below.
+# require_relative 'timbl'
+# require_relative 'mallet'
+# require_relative 'maxent'
+
+require_relative 'optimize'
+
+class Classifier
+
+  @@learners = [
+    ["timbl", "timbl", "Timbl"],
+    ["mallet", "mallet", "Mallet"],
+    ["maxent", "maxent", "Maxent"]
+  ]
+
+  def initialize(learner, params)
+
+    @ready = false
+
+    if params[0] == "optimise"
+      params.shift
+      @optimise = true
+    else
+      @optimise = false
+    end
+
+    program_path = ""
+    begin
+      program_path = params.shift.chomp
+      unless FileTest.exist? program_path
+        $stderr.puts "Error: Could not find classifier system at " + program_path
+        $stderr.puts "Perhaps an erroneous entry in your experiment file?"
+        exit 1
+      end
+    rescue NoMethodError
+      $stderr.puts "Error: No program path provided for classifier system."
+    end
+
+    # try to find our learner in the pre-set list of learners
+    learner_tuple = @@learners.assoc(learner)
+    unless learner_tuple
+      $stderr.puts "Error: I don't know the learner " + learner.to_s
+      $stderr.puts "Perhaps an erroneous entry in your experiment file?"
+      exit 1
+    end
+
+    # @todo AB: Investigate, why this dynamic require is necessary.
+    learner_name, learner_filename, learner_classname = learner_tuple
+    require_relative "#{learner_filename}"
+    @learner = eval(learner_classname).new(program_path,params)
+  end
+
+  # a classifier can (and has to be) either trained or read
+  def train(trainfile, classifier_file=nil)
+    # train on the training data in trainfile
+    # make sure we produce a valid file name
+
+    # it is possible to directly specify a filename for storing the classifier
+
+    trainfile.gsub!(/[<>]/,"")
+    trainfile.gsub!(/ /,"_")
+    if @optimise
+      STDERR.puts "[ML] using feature optimisation"
+      @optimiser = Optimise.new
+      @optimiser.init_from_data(trainfile)
+      optimisedfile = trainfile+".opted"
+      @optimiser.apply(trainfile,optimisedfile)
+      @learner.train(optimisedfile,classifier_file)
+      File.delete(optimisedfile)
+    else
+      STDERR.puts "[ML] no feature optimisation"
+      @learner.train(trainfile,classifier_file)
+    end
+    @ready = true
+  end
+
+
+  # returns true iff reading the classifier from the file has had success
+
+  def read(classifier_file)
+    # make sure we produce a valid file name
+    classifier_file.gsub!(/[<>]/,"")
+    classifier_file.gsub!(/ /,"_")
+
+    # read file, if present
+
+    status = @learner.read(classifier_file)
+
+    # if reading has failed, return "false"
+    unless status
+      STDERR.puts "reading from #{classifier_file} did not succeed"
+      return status
+    end
+
+    # read optimisation, if desired
+    if @optimise
+      optimisations_filename = Optimise.recommended_filename(classifier_file)
+      unless FileTest.exists? optimisations_filename
+        STDERR.puts "[ML] Error: attempted to read stored optimisation, but file does not exist"
+        return false
+      else
+        @optimiser = Optimise.new
+        @optimiser.init_from_file(optimisations_filename)
+      end
+    end
+
+    @ready = true
+    return true
+
+  end
+
+  # a classifier can be stored somewhere. This can be more than one file (classifier-specific),
+  # but all files start with "classifier_file"
+
+  def write(classifier_file)
+    # make sure we produce a valid file name
+    classifier_file.gsub!(/[<>]/,"")
+    classifier_file.gsub!(/ /,"_")
+    @learner.write(classifier_file)
+    if @optimise
+      @optimiser.store(Optimise.recommended_filename(classifier_file))
+    end
+  end
+
+  ###
+  # exists?
+  # check if a classifier is living at some particular path
+
+  def exists?(classifier_file)
+    classifier_file.gsub!(/[<>]/,"")
+    classifier_file.gsub!(/ /,"_")
+    return @learner.exists?(classifier_file)
+  end
+
+  # a classifier can be applied
+
+  # returns true iff application has had success
+
+  def apply(testfile,outfile) # test either on the training or the test data in the specified dir
+    # make sure we produce a valid file name
+    testfile.gsub!(/[<>]/,"")
+    testfile.gsub!(/ /,"_")
+    # make sure we produce a valid file name
+    outfile.gsub!(/[<>]/,"")
+    outfile.gsub!(/ /,"_")
+
+    unless @ready
+      STDERR.puts "[ML] Warning: learner not ready for testing! Must be trained or read."
+      return false
+    end
+
+    # do we have a testfile?
+    unless FileTest.exists?(testfile)
+      STDERR.puts "[ML] Warning: could not find testfile (maybe empty test set?)."
+      return false
+    end
+
+    if @optimise
+      optimisedfile = testfile+".opted"
+      @optimiser.apply(testfile,optimisedfile)
+      return @learner.apply(optimisedfile,outfile)
+      File.delete(optimisedfile)
+    else
+      return @learner.apply(testfile,outfile)
+    end
+  end
+
+  ###
+  # read classifier result file,
+  # returns a list of instance_results
+  # where an instance_result is a list of pairs [label, confidence]
+  # where the pairs are sorted by confidence
+  def read_resultfile(file)
+    return @learner.read_resultfile(file)
+  end
+end

data/lib/ml/mallet.rb

@@ -0,0 +1,236 @@
+# wrapper script for the Mallet toolkit Maxent classifier
+
+# Problem with Winnow: cannot be serialised (written to file). Support dropped.
+
+# sp 27 10 04
+
+
+require "tempfile"
+require "ftools"
+
+class Mallet
+
+  ###
+  def initialize(program_path,parameters)
+
+    if parameters.empty?
+      puts "Error: Mallet needs two paths (first the location of mallet itself and then the location of the interface, usually program/tools/mallet)."
+      puts "I got only the program path."
+      Kernel.exit
+    end
+
+    @malletpath = program_path
+    @interface_path = parameters.first
+    unless @malletpath =~ /\/$/
+      @malletpath = @malletpath + "/"
+    end
+
+    @learner = "MaxEnt,gaussianPriorVariance=1.0"
+
+    # classpath for mallet
+
+    @cp = "#{ENV["CLASSPATH"]}:#{@malletpath}class:#{@malletpath}lib/bsh.jar"
+
+  end
+
+  ###
+  def train(infilename,classifier_location)
+    csvfile = Tempfile.new(File.basename(infilename)+".csvtrain")
+    infile = File.new(infilename)
+    c45_to_csv(infile,csvfile) # training data in csv format
+    infile.close
+    csvfile.close
+    @mallet_train_vectors = infilename+".trainvectors" # training data in mallet format
+    if classifier_location
+      @classifier_mallet_path = classifier_location
+    else
+      @classifier_mallet_path = infilename+".classifier"
+    end
+
+    command1 = [@malletpath+"bin/csv2vectors ",
+                    " --input ",csvfile.path,
+                    " --output ",@mallet_train_vectors].join("")
+
+    command2 = ["cd #{@interface_path}; ",
+                "java -cp #{@cp} -Xmx1000m Train ",
+                " --train ",@mallet_train_vectors,
+                " --out ",@classifier_mallet_path,
+                " --trainer ",@learner].join("")
+#    STDERR.puts "[train 1] "+command1
+    successfully_run(command1) # encode
+#    STDERR.puts "[train 2] "+command2
+    successfully_run(command2) # train
+    csvfile.close(true)
+  end
+
+  def write(classifier_file)
+    if @classifier_mallet_path
+      %x{cp #{@classifier_mallet_path} #{classifier_file}.classifier} # store classifier
+   #    File.chmod(0664,classifier_file+".classifier")
+    end
+    if @mallet_train_vectors
+      %x{cp #{@mallet_train_vectors} #{classifier_file}.trainvectors} # store train vectors to recreate pipe for testing data
+#      File.chmod(0664,classifier_file+".trainvectors")
+    end
+  end
+
+  ###
+  def exists?(classifier_file)
+    return (FileTest.exists?(classifier_file+".trainvectors") and
+              FileTest.exists?(classifier_file+".classifier"))
+  end
+
+  ###
+  # return true iff reading the classifier has had success
+  def read(classifier_file)
+    @mallet_train_vectors = classifier_file+".trainvectors" # training data in mallet format
+    @classifier_mallet_path = classifier_file+".classifier"
+    unless FileTest.exists?(@mallet_train_vectors)
+      $stderr.puts "No classifier file "+@mallet_train_vectors
+      return false
+    end
+    unless FileTest.exists?(@classifier_mallet_path)
+      $stderr.puts "No classifier file "+@classifier_mallet_path
+      return false
+    end
+    return true
+  end
+
+  ###
+  def apply(infilename,outfilename)
+    unless @classifier_mallet_path and @mallet_train_vectors
+      return false
+    end
+
+    #    STDERR.puts "Testing on "+infilename
+    csvfile = Tempfile.new(File.basename(infilename)+".csvtest")
+
+    infile = File.new(infilename)
+    c45_to_csv(infile,csvfile) # training data in csv format
+    infile.close
+    csvfile.close
+
+    test_mallet_path = infilename+".test.vectors" # training data in mallet format
+
+    # $stderr.puts "test file in " + infilename
+    # $stderr.puts "using training vectors from " + @mallet_train_vectors
+
+    # copy train vectors to temp file.
+    # reason: mallet in std edition reads _and writes_ this file
+    # if rosy is interrupted, corrupted (ie incomplete) train vector files
+    # result
+
+    tempfile = Tempfile.new("mallet")
+    tempfilename = tempfile.path
+    unless File.copy(@mallet_train_vectors,tempfilename)
+      return false
+    end
+
+    command1 = [@malletpath+"bin/csv2vectors", # encode testing data
+                " --input ",csvfile.path,
+                " --output ",test_mallet_path,
+                " --use-pipe-from ",tempfilename].join("")
+
+#    $stderr.puts "Mallet encode: " + command1
+    unless successfully_run(command1) # encode
+      return false
+    end
+
+    File.safe_unlink(tempfilename)
+
+    # some error in encoding?
+    unless FileTest.exists?(test_mallet_path)
+      return false
+    end
+
+    command2 = ["cd #{@interface_path}; ",
+                "java -cp #{@cp} -Xmx1000m Classify ",
+                @classifier_mallet_path," ",
+                test_mallet_path," ",
+                "> ",outfilename].join("")
+
+    # classify
+#    $stderr.puts "Mallet classify: " + command2
+    unless    successfully_run(command2)
+      return false
+    end
+
+    # some error in classification
+    unless FileTest.exists?(outfilename)
+      return false
+    end
+
+     # no errors = success
+    csvfile.close(true)
+    return true
+  end
+
+  #####
+  # format of Mallet result file:
+  # <best label> <confidence> \t <secondbest_label> <confidence>....
+  def read_resultfile(filename)
+    begin
+      f = File.new(filename)
+    rescue
+      $stderr.puts "Mallet error: cannot read Mallet result file #{filemame}."
+      return nil
+    end
+
+    retv = []
+
+    f.each { |line|
+      line_results = []
+      pieces = line.split
+
+      while not(pieces.empty?)
+        label = pieces.shift
+
+        begin
+          confidence = pieces.shift.to_f
+        rescue
+          $stderr.puts "Error reading mallet output: invalid line: #{line}"
+          confidence = 0
+        end
+
+        line_results << [label, confidence]
+      end
+      retv << line_results
+    }
+
+    return retv
+  end
+
+
+  ###################################
+  private
+
+  ###
+  # mallet needs "comma separated values"-file
+  # input: features separated by comma
+  # output:
+  # line_number classlabel features_joined_by_spaces
+  def c45_to_csv(inpipe,outpipe)
+    idx = 0
+    while (line = inpipe.gets)
+      line.chomp!
+      idx += 1
+      la = line.split(",")
+      label = la.pop
+      if label[-1,1] == "."
+        label.chop!
+      end
+      outpipe.puts [idx,label].join(" ")+" "+la.join(" ")
+    end
+  end
+
+  ###
+  def successfully_run(command)
+    retv = Kernel.system(command)
+    unless retv
+      $stderr.puts "Error running classifier. Continuing."
+      $stderr.puts "Offending command: "+command
+ #     exit 1
+    end
+    return retv
+  end
+end