treat 1.2.0 → 2.0.0rc1

data/lib/treat/workers/{group.rb → groupable.rb}

@@ -1,10 +1,10 @@
-module Treat::Workers::Group
+module Treat::Workers::Groupable
 
   # Lazily load the worker classes in the group.
   def const_missing(const)
     bits = self.ancestors[0].to_s.split('::')
-    bits.collect! { |bit| ucc(bit) }
-    file = bits.join('/') + "/#{ucc(const)}"
+    bits.collect! { |bit| bit.ucc }
+    file = bits.join('/') + "/#{const.ucc}"
     if not File.readable?(Treat.paths.lib + "#{file}.rb")
       raise Treat::Exception,
       "File '#{file}.rb' corresponding to " +
@@ -26,7 +26,7 @@ module Treat::Workers::Group
   # Populates once the list of the workers in the group
   # by crawling the filesystem.
   def list
-    mod = ucc(cl(self))
+    mod = self.mn.ucc
     if @@list[mod].nil?
       @@list[mod] = []
       dirs = Dir[Treat.paths.lib + "treat/workers/*/#{mod}/*.rb"]
@@ -43,7 +43,7 @@ module Treat::Workers::Group
   def has_target?(target, strict = false)
     is_target = false
     self.targets.each do |entity_type|
-      t = cc(entity_type)
+      t = entity_type.cc
       entity_type = Treat::Entities.const_get(t)
       if target < entity_type ||
         entity_type == target
@@ -57,10 +57,10 @@ module Treat::Workers::Group
   # the algorithm is added, it will be automatically
   # installed on all the targets of the group.
   def add(class_name, &block)
-    c = cc(class_name).intern
+    c = class_name.cc.intern
     klass = self.const_set(c, Class.new)
     method = self.method
-    @@list[ucc(cl(self))] << class_name
+    @@list[self.mn.ucc] << class_name
     klass.send(:define_singleton_method,
     method) do |entity, options={}|
       block.call(entity, options)
@@ -75,7 +75,7 @@ module Treat::Workers::Group
 
   # Modify the extended class.
   def self.extended(group)
-    
+
     group.module_eval do
       
       class << self
@@ -114,7 +114,7 @@ module Treat::Workers::Group
       @method = nil
       def self.method
         return @method if @method
-        m = ucc(cl(self)).dup
+        m = self.mn.ucc.dup
         if  m[-4..-1] == 'zers'
           if type == :annotator
             m[-5..-1] = m[-6] == 'l' ? '' : 'y'

data/lib/treat/workers/inflectors/cardinalizers/linguistics.rb

@@ -3,9 +3,7 @@
 # number in words in cardinal form.
 #
 # Project website: http://deveiate.org/projects/Linguistics/
-module Treat::Workers::Inflectors::Cardinalizers::Linguistics
-  
-  require 'treat/loaders/linguistics'
+class Treat::Workers::Inflectors::Cardinalizers::Linguistics
   
   DefaultOptions = {
     :language => Treat.core.language.default

data/lib/treat/workers/inflectors/conjugators/linguistics.rb

@@ -4,18 +4,16 @@
 # Project website: http://deveiate.org/projects/Linguistics/
 module Treat::Workers::Inflectors::Conjugators::Linguistics
 
-  require 'treat/loaders/linguistics'
-  
   DefaultOptions = {
     :strict => false
   }
   
   Forms = {
-    :present_participle =>
+    'present_participle' =>
     {:mode => 'participle', :tense => 'present'},
-    :infinitive => {:mode => 'infinitive'},
-    :plural_verb => {:count => 'plural'},
-    :singular_verb => {:count => 'singular'}
+    'infinitive' => {:mode => 'infinitive'},
+    'plural_verb' => {:count => 'plural'},
+    'singular_verb' => {:count => 'singular'}
   }
   
   # Conjugate a verb using ruby linguistics with the specified
@@ -35,7 +33,7 @@ module Treat::Workers::Inflectors::Conjugators::Linguistics
     cat = entity.check_has(:category)
     return if cat != 'verb' && options[:strict]
 
-    options = Forms[options[:form]] if options[:form]
+    options = Forms[options[:form].to_s] if options[:form]
 
     klass = Treat::Loaders::Linguistics.load(entity.language)
     if options[:mode] == 'infinitive'

data/lib/treat/workers/inflectors/declensors/english.rb

@@ -1,38 +1,31 @@
-# This class is a wrapper for the Inflect module,
-# copied from the unmaintained 'english' ruby gem,
-# created by Thomas Sawyer.
+# Inflection using the inflect module copied from 
+# the unmaintained 'english' ruby gem.
 #
-# Released under the MIT License.
-#
-#  http://english.rubyforge.org
+# License: MIT
+# Website: http://english.rubyforge.org
 class Treat::Workers::Inflectors::Declensors::English
 
-  require 'treat/inflectors/declensors/english/inflect'
+  require_relative 'english/inflect'
+  
+  # Part of speech that can be declensed.
+  POS = ['noun', 'adjective', 'determiner']
   
   # Retrieve the declensions (singular, plural)
   # of an english word using a class lifted from
   # the 'english' ruby gem.
   def self.declense(entity, options)
-    
     cat = entity.check_has(:category)
-    unless ['noun', 'adjective', 'determiner'].
-      include?(cat)
-        return
-    end
-    
+    return unless POS.include?(cat)
     unless options[:count]
-      raise Treat::Exception,
-      "Must supply option count (:singular or :plural)."
+      raise Treat::Exception, 'Must supply ' +
+      'option count ("singular" or "plural").'
     end
-    
     string = entity.to_s
-
-    if options[:count] == :plural
+    if options[:count].to_s == 'plural'
       Inflect.plural(string)
-    elsif options[:count] == :singular
+    elsif options[:count].to_s == 'singular'
       Inflect.singular(string)
     end
-    
   end
   
 end

data/lib/treat/workers/inflectors/declensors/linguistics.rb

@@ -1,49 +1,50 @@
-# This class is a wrapper for the functions included
-# in the 'linguistics' gem that allow to obtain the
-# declensions of a word.
+# Inflection using the the 'linguistics' gem (will attempt
+# to load the Linguistics module corresponding to the
+# language of the entity.
 #
-# Project website: http://deveiate.org/projects/Linguistics/
+# Website: http://deveiate.org/projects/Linguistics/
 class Treat::Workers::Inflectors::Declensors::Linguistics
 
-  require 'treat/loaders/linguistics'
-
+  # Part of speech that can be declensed.
+  POS = ['noun', 'adjective', 'determiner']
+  
   # Retrieve a declension of a word using the 'linguistics' gem.
   #
   # Options:
   #
   # - (Identifier) :count => :singular, :plural
   def self.declense(entity, options = {})
-    
-    cat = entity.check_has(:category)
-    unless ['noun', 'adjective', 'determiner'].
-      include?(cat)
-        return
-    end
-    
+
+    cat = entity.get(:category)
+    return if cat && !POS.include?(cat)
     unless options[:count]
-      raise Treat::Exception,
-      "Must supply option count (:singular or :plural)."
+      raise Treat::Exception, 'Must supply ' +
+      ':count option ("singular" or "plural").'
     end
     
-    klass = Treat::Loaders::Linguistics.load(entity.language)
+    klass = Treat::Loaders::
+    Linguistics.load(entity.language)
     string = entity.to_s
-    
-    if options[:count] == 'plural'
-      
-      if entity.has?(:category) &&
-        ['noun', 'adjective', 'verb'].
-        include?(entity.category)
+
+    if options[:count].to_s == 'plural'
+      if (entity.has?(:category))
+        result = ''
         silence_warnings do
-          klass.send(
-          :"plural_#{entity.category}", 
+          result = klass.send(
+          :"plural_#{entity.category}",
           string)
         end
+        return result
       else
-        klass.plural(string)
+        return klass.plural(string)
       end
-      
+
+    else
+      raise Treat::Exception,
+      "Ruby Linguistics does not support " +
+      "singularization of words."
     end
-    
+
   end
 
-end
+end

data/lib/treat/workers/inflectors/ordinalizers/linguistics.rb

@@ -5,8 +5,6 @@
 # Project website: http://deveiate.org/projects/Linguistics/
 class Treat::Workers::Inflectors::Ordinalizers::Linguistics
 
-  require 'treat/loaders/linguistics'
-  
   DefaultOptions = {
     :language => Treat.core.language.default
   }

data/lib/treat/workers/inflectors/stemmers/porter.rb

@@ -1,15 +1,13 @@
-# Stem a word using a native Ruby implementation of the
-# Porter stemming algorithm, ported to Ruby from a
-# version coded up in Perl. This is a simplified
-# implementation; for a true and fast Porter stemmer,
-# see Treat::Workers::Inflectors::Stemmers::PorterC.
+# Stemming using a native Ruby implementation of the
+# Porter stemming algorithm, a rule-based suffix-stripping
+# stemmer which is very widely used and is considered the 
+# de-facto standard algorithm used for English stemming.
 #
-# Authored by Ray Pereda (raypereda@hotmail.com).
-# Unknown license.
+# Original paper: Porter, 1980. An algorithm for suffix 
+# stripping. Program, vol. 14, no. 3, p. 130-137.
 #
-# Original paper: Porter, 1980. An algorithm for suffix stripping,
-# Program, Vol. 14, no. 3, pp 130-137,
-# Original C implementation: http://www.tartarus.org/~martin/PorterStemmer.
+# Authors: Ray Pereda (raypereda@hotmail.com).
+# License: Unknown.
 class Treat::Workers::Inflectors::Stemmers::Porter
   
   # Returns the stem of a word using a native Porter stemmer.

data/lib/treat/workers/inflectors/stemmers/porter_c.rb

@@ -1,11 +1,11 @@
-# Stems words using the 'ruby-stemmer' gem, which
-# wraps a C version of the Porter stemming algorithm.
+# Stemming using a wrapper for a C implementation of the
+# Porter stemming algorithm, a rule-based suffix-stripping
+# stemmer which is very widely used and is considered the 
+# de-facto standard algorithm used for English stemming.
 #
-# Project website: https://github.com/aurelian/ruby-stemmer
-# Original paper: Porter, 1980. An algorithm for suffix stripping,
-# Program, Vol. 14, no. 3, pp 130-137,
-# Original C implementation: http://www.tartarus.org/~martin/PorterStemmer.
-module Treat::Workers::Inflectors::Stemmers::PorterC
+# Original paper: Porter, 1980. An algorithm for suffix 
+# stripping. Program, vol. 14, no. 3, p. 130-137.
+class Treat::Workers::Inflectors::Stemmers::PorterC
   
   # Require the 'ruby-stemmer' gem.
   silence_warnings { require 'lingua/stemmer' }

data/lib/treat/workers/inflectors/stemmers/uea.rb

@@ -1,15 +1,10 @@
-# Stems a word using the UEA algorithm, implemented
-# by the 'uea-stemmer' gem.
-#
-# "Similar to other stemmers, UEA-Lite operates on a
-# set of rules which are used as steps. There are two
+# Stemming using the UEA algorithm, a stemmer that operates
+# on a set of rules which are used as steps. There are two
 # groups of rules: the first to clean the tokens, and
-# the second to alter suffixes."
+# the second to alter suffixes.
 #
-# Project website: https://github.com/ealdent/uea-stemmer
 # Original paper: Jenkins, Marie-Claire, Smith, Dan,
 # Conservative stemming for search and indexing, 2005.
-# http://www.uea.ac.uk/polopoly_fs/1.85493!stemmer25feb.pdf
 class Treat::Workers::Inflectors::Stemmers::UEA
 
   # Require the 'uea-stemmer' gem.

data/lib/treat/workers/learners/classifiers/id3.rb

@@ -1,3 +1,10 @@
+# Classification based on Ross Quinlan's ID3 
+# (Iterative Dichotomiser 3) decision tree 
+# algorithm.
+# 
+# Original paper: Quinlan, J. R. 1986. 
+# Induction of Decision Trees. Mach. Learn. 
+# 1, 1 (Mar. 1986), 81-106.
 class Treat::Workers::Learners::Classifiers::ID3
   
   require 'decisiontree'
@@ -5,24 +12,20 @@ class Treat::Workers::Learners::Classifiers::ID3
   @@classifiers = {}
   
   def self.classify(entity, options = {})
-    
-    set = options[:training]
-    cl = set.problem
-    
-    if !@@classifiers[cl]
+    dset = options[:training]
+    prob = dset.problem
+    if !@@classifiers[prob]
       dec_tree = DecisionTree::ID3Tree.new(
-      cl.feature_labels.map { |l| l.to_s }, 
-      set.items.map { |i| i[:features]}, 
-      cl.question.default, cl.question.type)
+      prob.feature_labels.map { |l| l.to_s }, 
+      dset.items.map { |i| i[:features] }, 
+      prob.question.default, prob.question.type)
       dec_tree.train
-      @@classifiers[cl] = dec_tree
+      @@classifiers[prob] = dec_tree
     else
-      dec_tree = @@classifiers[cl]
-      dec_tree.graph('testingbitch')
+      dec_tree = @@classifiers[prob]
     end
-    dec_tree.predict(
-      cl.export_features(entity, false)
-    )
+    vect = prob.export_features(entity, false)
+    dec_tree.predict(vect)
   end
   
 end

data/lib/treat/workers/learners/classifiers/linear.rb

@@ -11,35 +11,23 @@ class Treat::Workers::Learners::Classifiers::Linear
   }
   
   def self.classify(entity, options = {})
-    
     options = DefaultOptions.merge(options)
-    set = options[:training]
-    problem = set.problem
-    
-    if !@@classifiers[problem]
-      labels = problem.question.labels
-      unless labels
-        raise Treat::Exception,
-        "LibLinear requires that you provide the possible " +
-        "labels to assign to classification items when " +
-        "specifying the question."
-      end
-      param = LParameter.new
-      param.solver_type = options[:solver_type]
-      param.eps = options[:eps]
-      bias = options[:bias]
-      data = set.items.map do |item| 
-        self.array_to_hash(item[:features]) 
-      end
-      prob = LProblem.new(labels, data, bias)
-      @@classifiers[problem] = 
-      LModel.new(prob, param)
+    dset = options[:training]
+    prob, items = dset.problem, dset.items
+    if !@@classifiers[prob]
+      lparam = LParameter.new
+      lparam.solver_type = options[:solver_type]
+      lparam.eps = options[:eps]
+      lbls = items.map { |it| it[:features][-1] }
+      exs = items.map { |it| it[:features][0..-2] }.
+      map { |ary| self.array_to_hash(ary) }
+      lprob = LProblem.new(lbls, exs, options[:bias])
+      model = LModel.new(lprob, lparam)
+      @@classifiers[prob] = model
     end
-
-    @@classifiers[problem].predict(
-    self.array_to_hash(problem.
-    export_features(entity, false)))
-    
+    features = prob.export_features(entity, false)
+    @@classifiers[prob].predict(
+    self.array_to_hash(features))
   end
   
   def self.array_to_hash(array)

data/lib/treat/workers/learners/classifiers/mlp.rb

@@ -1,30 +1,43 @@
-# Currently, this MLP is limited to 1 output.
+# Classification based on a multilayer perceptron.
 class Treat::Workers::Learners::Classifiers::MLP
   
-  require 'ai4r'
+  require 'ruby_fann/neural_network'
   
-  @@mlps = {}
+  DefaultOptions = {
+    num_inputs: 3,
+    hidden_neurons: [2, 8, 4, 3, 4],
+    num_outputs: 1,
+    max_neurons: 1000,
+    neurons_between_reports: 1,
+    desired_error: 0.1
+  }
+  
+  @@classifiers = {}
   
   def self.classify(entity, options = {})
-    
-    set = options[:training]
-    cl = set.problem
-      
-    if !@@mlps[cl]
-      net = Ai4r::NeuralNetwork::Backpropagation.new(
-      [cl.feature_labels.size, 3, 1])
-      set.items.each do |item|
-        inputs = item[:features][0..-2]
-        outputs = [item[:features][-1]]
-        net.train(inputs, outputs)
+    options = DefaultOptions.merge(options)
+    dset = options[:training]
+    prob, items = dset.problem, dset.items
+    if !@@classifiers[prob]
+      fann = RubyFann::Standard.new(options)
+      inputs = items.map { |it| it[:features][0..-2] }
+      outputs = items.map { |it| [it[:features][-1]] }
+      training = silence_stdout do 
+        RubyFann::TrainData.new(inputs: 
+        inputs, desired_outputs: outputs)
       end
-      @@mlps[cl] = net
+      params = [options[:max_neurons],
+      options[:neurons_between_reports],
+      options[:desired_error]]
+      fann.train_on_data(training, *params)
+      @@classifiers[prob] = fann
     else
-      net = @@mlps[cl]
+      fann = @@classifiers[prob]
     end
-    
-    net.eval(cl.export_features(entity, false))[0]
-    
+    vect = prob.export_features(entity, false)
+    Treat.core.verbosity.silence ? 
+    silence_stdout { fann.run(vect)[0] } :
+    fann.run(vect)[0]
   end
   
 end

data/lib/treat/workers/learners/classifiers/svm.rb

@@ -1,31 +1,38 @@
 class Treat::Workers::Learners::Classifiers::SVM
   
-  require 'svm'
+  require 'libsvm'
   
   @@classifiers = {}
+
+  DefaultOptions = {
+    cache_size: 1, # in MB
+    eps: 0.001,
+    c: 10
+  }
   
+  # - (Numeric) :cache_size => cache size in MB.
+  # - (Numeric) :eps => tolerance of termination criterion
+  # - (Numeric) :c => C parameter
   def self.classify(entity, options = {})
-    
-    set = options[:training]
-    problem = set.problem
-    
-    if !@@classifiers[problem]
-      labels = problem.question.labels
-      unless labels
-        raise Treat::Exception,
-        "LibSVM requires that you provide the possible " +
-        "labels to assign to classification items when " +
-        "specifying the question."
-      end
-      data = set.items.map  { |item| item[:features] }
-      prob = Problem.new(labels, data)
-      param = Parameter.new(:kernel_type => LINEAR, :C => 10)
-      @@classifiers[problem] = Model.new(prob, param)
+    options = DefaultOptions.merge(options)
+    dset = options[:training]
+    prob, items = dset.problem, dset.items
+    if !@@classifiers[prob]
+      lprob = Libsvm::Problem.new
+      lparam = Libsvm::SvmParameter.new
+      lparam.cache_size = options[:cache_size]
+      lparam.eps = options[:eps]
+      lparam.c = options[:c]
+      llabels = items.map { |it| it[:features][-1] }
+      lexamples = items.map { |it| it[:features][0..-2] }.
+      map { |ary| Libsvm::Node.features(ary) }
+      lprob.set_examples(llabels, lexamples)
+      model = Libsvm::Model.train(lprob, lparam)
+      @@classifiers[prob] = model
     end
-
-    @@classifiers[problem].predict_probability(
-    problem.export_features(entity, false))[0]
-    
+    features = prob.export_features(entity, false)
+    @@classifiers[prob].predict(
+    Libsvm::Node.features(features))
   end
   
 end

data/lib/treat/workers/lexicalizers/categorizers/from_tag.rb

@@ -1,6 +1,22 @@
-# Finds the general part of speech of an entity
-# (:sentence, :noun_phrase, :verb, :adverb, etc.)
-# from its tag (e.g. 'S', 'NP', 'VBZ', 'ADV', etc.).
+# Uses a hand-made table to convert part-of-speech tags
+# to descriptive English word or phrase categories.
+# 
+# Original paper: 1) Manning, Christopher and Schütze, Hinrich,
+# 1999. Foundations of Statistical Natural Language
+# Processing. MIT Press, p. 141-142. 
+# 
+# 2) Schiller et al. 1999. Guidelines fur das Tagging 
+# deutscher Textcorpora. Universitat Tubingen. Institut
+# fur maschinelle Sprachverarbeitung.
+# 
+# 3) Anne Abeille, Francois Toussenel, et Martine
+# Cheradame. 2004. Corpus Le Monde: Annotations en 
+# constituants. Guide pour les correcteurs. LLF et 
+# UFRL and Universite Paris 7.
+
+# 4) Xue and Palmer. 2008. Annotating the Propositions in 
+# the Penn Chinese Treebank. University of Pennsylvania,
+# Department of Computer Information and Science.
 class Treat::Workers::Lexicalizers::Categorizers::FromTag
 
   Pttc = Treat.tags.aligned.phrase_tags_to_category

data/lib/treat/workers/lexicalizers/sensers/wordnet.rb

@@ -1,8 +1,13 @@
-# Obtain lexical information about a word using the
-# ruby 'wordnet' gem.
+# Sense information (synonyms, antonyms, hypernyms
+# and hyponyms) obtained through a Ruby parser that
+# accesses Wordnet flat files.
+# 
+# Original paper: George A. Miller (1995). WordNet: 
+# A Lexical Database for English. Communications of 
+# the ACM Vol. 38, No. 11: 39-41.
 class Treat::Workers::Lexicalizers::Sensers::Wordnet
 
-  # Require the 'wordnet' gem.
+  # Require the 'wordnet' gem (install as 'rwordnet').
   require 'wordnet'
   
   # Patch for bug.
@@ -13,7 +18,7 @@ class Treat::Workers::Lexicalizers::Sensers::Wordnet
   end
   
   # Require an adaptor for Wordnet synsets.
-  require 'treat/workers/lexicalizers/sensers/wordnet/synset'
+  require_relative 'wordnet/synset'
   
   # Noun, adjective and verb indexes.
   @@indexes = {}
@@ -45,13 +50,16 @@ class Treat::Workers::Lexicalizers::Sensers::Wordnet
     
     lemma.synsets.each do |synset|
       synsets << 
-      Treat::Workers::Lexicalizers::Sensers::Wordnet::Synset.new(synset)
+      Treat::Workers::Lexicalizers::
+      Sensers::Wordnet::Synset.new(synset)
     end
     
     ((synsets.collect do |ss|
       ss.send(options[:nym])
-    end - [word.value]).flatten).uniq
-    
+    end - [word.value]).
+    flatten).uniq.map do |token|
+      token.gsub('_', ' ')
+    end
   end
 
 end

data/lib/treat/workers/lexicalizers/taggers/brill/patch.rb

@@ -11,7 +11,10 @@ rescue TypeError
     patch = true
     # Unset the class Word for the duration
     # of loading the tagger.
-    Object.const_unset(:Word); retry
+    Object.instance_eval do 
+      remove_const(:Word)
+    end
+    retry
   else
     raise Treat::Exception,
     'Something went wrong due to a name clash with the "rbtagger" gem.' +