treat 1.2.0 → 2.0.0rc1

data/lib/treat/{core → learning}/data_set.rb

@@ -2,7 +2,7 @@
 # problem as well as data for entities that
 # have already been classified, complete with
 # references to these entities.
-class Treat::Core::DataSet
+class Treat::Learning::DataSet
   
   # The classification problem this
   # data set holds data for.
@@ -13,26 +13,26 @@ class Treat::Core::DataSet
   
   # Initialize the DataSet.
   def initialize(problem)
-    unless problem.is_a?(Treat::Core::Problem)
+    unless problem.is_a?(Treat::Learning::Problem)
       raise Treat::Exception, "The first argument " +
       "to initialize should be an instance of " +
-      "Treat::Core::Problem."
+      "Treat::Learning::Problem."
     end
     @problem, @items = problem, []
   end
   
   def self.build(from)
     if from.is_a?(Hash)
-      Treat::Core::DataSet.unserialize(
+      Treat::Learning::DataSet.unserialize(
       Treat.databases.default.adapter, from)
     elsif from.is_a?(String)
       unless File.readable?(from)
         raise Treat::Exception,
-        "Attempting to initialize data set from "
-        "file #{from}, but it is not readable."
+        "Attempting to initialize data set from " +
+        "file '#{from}', but it is not readable."
       end
-      Treat::Core::DataSet.unserialize(
-      extension, file: from)
+      Treat::Learning::DataSet.unserialize(
+      File.extname(from)[1..-1], file: from)
     end
   end
   
@@ -92,7 +92,7 @@ class Treat::Core::DataSet
       next unless tag.proc_string
       tag.proc = eval(tag.proc_string)
     end
-    data_set = Treat::Core::DataSet.new(problem)
+    data_set = Treat::Learning::DataSet.new(problem)
     data_set.items = items
     data_set
   end
@@ -131,7 +131,7 @@ class Treat::Core::DataSet
       raise Treat::Exception, 
       "Couldn't retrieve problem ID #{options[:problem]}."
     end
-    problem = Treat::Core::Problem.from_hash(p_record)
+    problem = Treat::Learning::Problem.from_hash(p_record)
     data = database.collection('data').find(options).to_a
     items = []
     data.each do |datum|
@@ -142,7 +142,7 @@ class Treat::Core::DataSet
       item[:id] = datum['id']
       items << item
     end
-    data_set = Treat::Core::DataSet.new(problem)
+    data_set = Treat::Learning::DataSet.new(problem)
     data_set.items = items
     data_set
   end

data/lib/treat/{core → learning}/export.rb

@@ -1,6 +1,6 @@
 # Represents a feature to be used
 # in a classification task.
-class Treat::Core::Export
+class Treat::Learning::Export
   
   # The name of the feature. If no 
   # proc is supplied, this assumes
@@ -55,5 +55,5 @@ class Treat::Core::Export
   
 end
 
-class Treat::Core::Feature < Treat::Core::Export; end
-class Treat::Core::Tag < Treat::Core::Export; end
+class Treat::Learning::Feature < Treat::Learning::Export; end
+class Treat::Learning::Tag < Treat::Learning::Export; end

data/lib/treat/{core → learning}/problem.rb

@@ -2,7 +2,7 @@
 # - What question are we trying to answer?
 # - What features are we going to look at
 #   to attempt to answer that question?
-class Treat::Core::Problem
+class Treat::Learning::Problem
 
   # A unique identifier for the problem.
   attr_accessor :id
@@ -20,21 +20,21 @@ class Treat::Core::Problem
   # Initialize the problem with a question
   # and an arbitrary number of features.        # FIXME: init with id!?
   def initialize(question, *exports)
-    unless question.is_a?(Treat::Core::Question)
+    unless question.is_a?(Treat::Learning::Question)
       raise Treat::Exception,
       "The first argument to initialize " +
       "should be an instance of " +
-      "Treat::Core::Question."
+      "Treat::Learning::Question."
     end
-    if exports.any? { |f| !f.is_a?(Treat::Core::Export) }
+    if exports.any? { |f| !f.is_a?(Treat::Learning::Export) }
       raise Treat::Exception,
       "The second argument and all subsequent ones " +
       "to initialize should be instances of subclasses " +
-      "of Treat::Core::Export."
+      "of Treat::Learning::Export."
     end
     @question, @id = question, object_id
     @features = exports.select do |exp|
-      exp.is_a?(Treat::Core::Feature)
+      exp.is_a?(Treat::Learning::Feature)
     end
     if @features.size == 0
       raise Treat::Exception, 
@@ -42,7 +42,7 @@ class Treat::Core::Problem
       "one feature to work with."
     end
     @tags = exports.select do |exp|
-      exp.is_a?(Treat::Core::Tag)
+      exp.is_a?(Treat::Learning::Tag)
     end
     @feature_labels = @features.map { |f| f.name }
     @tag_labels = @tags.map { |t| t.name }
@@ -63,7 +63,7 @@ class Treat::Core::Problem
   # all of the features.
   def export_features(e, include_answer = true)
     features = export(e, @features)
-    return features unless include_answer
+    return features if !include_answer
     features << (e.has?(@question.name) ? 
     e.get(@question.name) : @question.default)
     features
@@ -80,9 +80,11 @@ class Treat::Core::Problem
 
   def export(entity, exports)
     unless @question.target == entity.type
+      targ, type = @question.target, entity.type
       raise Treat::Exception, 
-      "This classification problem targets #{@question.target}s, " +
-      "but a(n) #{entity.type} was passed to export instead."
+      "This classification problem targets " +
+      "#{targ}s, but a(n) #{type} " +
+      "was passed to export instead."
     end
     ret = []
     exports.each do |export|
@@ -103,28 +105,36 @@ class Treat::Core::Problem
     'id' => @id }
   end
   
+  def object_to_hash(obj)
+    hash = {}
+    obj.instance_variables.each do |var|
+      val = obj.instance_variable_get(var)
+      hash[var.to_s.delete("@")] = val
+    end
+    hash
+  end
+  
   def self.from_hash(hash)
-    question = Treat::Core::Question.new(
+    question = Treat::Learning::Question.new(
       hash['question']['name'], 
       hash['question']['target'],
-      hash['question']['type'],
       hash['question']['default'],
-      hash['question']['labels']
+      hash['question']['type']
     )
     features = []
     hash['features'].each do |feature|
-      features << Treat::Core::Feature.new(
+      features << Treat::Learning::Feature.new(
       feature['name'], feature['default'],
       feature['proc_string'])
     end
     tags = []
     hash['tags'].each do |tag|
-      tags << Treat::Core::Tag.new(
+      tags << Treat::Learning::Tag.new(
       tag['name'], tag['default'],
       tag['proc_string'])
     end
     features_and_tags = features + tags
-    p = Treat::Core::Problem.new(question, *features_and_tags)
+    p = Treat::Learning::Problem.new(question, *features_and_tags)
     p.id = hash['id']
     p
   end

data/lib/treat/{core → learning}/question.rb

@@ -1,6 +1,6 @@
 # Defines a question to answer in the
 # context of a classification problem.
-class Treat::Core::Question
+class Treat::Learning::Question
   
   # Defines an arbitrary label for the
   # question we are trying to answer 
@@ -16,12 +16,9 @@ class Treat::Core::Question
   attr_reader :type
   # Default for the answer to the question.
   attr_reader :default
-  # A list of possible answers to the question.
-  attr_reader :labels
   
   # Initialize the question.
-  def initialize(name, target, 
-    type = :continuous, default = nil, labels = [])
+  def initialize(name, target, default = nil, type = :continuous)
     unless name.is_a?(Symbol)
       raise Treat::Exception, 
       "Question name should be a symbol."
@@ -35,8 +32,8 @@ class Treat::Core::Question
       raise Treat::Exception, "Type should be " +
       "continuous or discrete."
     end
-    @name, @target, @type, @default, @labels = 
-     name,  target,  type,  default,  labels
+    @name, @target, @type, @default = 
+     name,  target,  type,  default
   end
 
   # Custom comparison operator for questions.
@@ -44,8 +41,7 @@ class Treat::Core::Question
     @name == question.name &&
     @type == question.type &&
     @target == question.target &&
-    @default == question.default &&
-    @labels = question.labels
+    @default == question.default
   end
   
 end

data/lib/treat/loaders/linguistics.rb

@@ -1,10 +1,8 @@
 # A helper class to load a language class
-# registered with the Linguistics gem.
+# registered with the Linguistics gem, for
+# example Linguistics::EN.
 class Treat::Loaders::Linguistics
 
-  # Linguistics throws warnings; silence them.
-  silence_warnings { require 'linguistics' }
-
   # Linguistics classes for each language.
   @@languages = {}
 
@@ -13,16 +11,17 @@ class Treat::Loaders::Linguistics
   # if there is no such language class registered.
   def self.load(language)
     silence_warnings do
+      # Linguistics throws warnings; silence them.
+      silence_warnings { require 'linguistics' }
+      code = language.to_s[0..1].upcase
       @@languages[language] ||= 
-      ::Linguistics.const_get(
-      language.to_s[0..1].upcase)
+      ::Linguistics.const_get(code)
     end
     return @@languages[language]
   rescue RuntimeError
     raise Treat::Exception,
-    "Ruby Linguistics does " +
-    "not have a module installed " +
-    "for the #{language} language."
+    "Ruby Linguistics does not have a module " +
+    "installed for the #{language} language."
   end
 
 end

data/lib/treat/loaders/stanford.rb

@@ -1,30 +1,24 @@
 # A helper class to load the CoreNLP package.
 class Treat::Loaders::Stanford
-
-  require 'stanford-core-nlp'
   
+  # Keep track of whether its loaded or not.
   @@loaded = false
 
   # Load CoreNLP package for a given language.
   def self.load(language = nil)
     return if @@loaded
+    require 'stanford-core-nlp'
     language ||= Treat.core.language.default
-    
     StanfordCoreNLP.jar_path = 
     Treat.libraries.stanford.jar_path || 
     Treat.paths.bin + 'stanford/'
-    
     StanfordCoreNLP.model_path = 
     Treat.libraries.stanford.model_path || 
     Treat.paths.models + 'stanford/'
-    
     StanfordCoreNLP.use(language)
-    if Treat.core.verbosity.silence
-      StanfordCoreNLP.log_file = NULL_DEVICE
-    end
-    
-    StanfordCoreNLP.bind
-    @@loaded = true
+    StanfordCoreNLP.log_file = '/dev/null' if 
+    Treat.core.verbosity.silence
+    StanfordCoreNLP.bind; @@loaded = true
   end
   
 end

data/lib/treat/modules.rb

@@ -0,0 +1,33 @@
+module Treat
+  
+  # Contains common utility/helper functions.
+  module Helpers; include Autoload; end
+  
+  # Contains all the configuration options.
+  module Config; include Autoload; end
+  
+  # Import all the configuration options.
+  Treat::Config.import!
+
+  # Contains classes to load external libraries.
+  module Loaders; include Autoload; end
+
+  # Contains machine learning core classes.
+  module Learning; include Autoload; end
+
+  # Contains the document object models.
+  module Entities; include Autoload; end
+
+  # Contains all the worker categories.
+  module Workers; include Autoload; end
+  
+  # Make all the worker categories.
+  Treat::Workers.categorize!
+  
+  # Installs builders on core Ruby objects.
+  module Proxies; include Autoload; end
+  
+  # Core classes (installer, server, etc.)
+  module Core; include Autoload; end
+  
+end

data/lib/treat/proxies/array.rb

@@ -0,0 +1,27 @@
+module Treat::Proxies
+  
+  module Array
+    # Include base proxy functionality.
+    include Treat::Proxies::Proxy
+    def method_missing(sym, *args, &block)
+      if [:do, :apply].include?(sym) || 
+        Treat::Workers.lookup(sym)
+        map do |el|
+          if el.is_a?(Treat::Entities::Entity)
+            el.send(sym, *args)
+          else
+            el.to_entity.send(sym, *args)
+          end
+        end
+      else
+        super(sym, *args, &block)
+      end
+    end
+  end
+  
+  # Include Treat methods on numerics.
+  ::Array.class_eval do
+    include Treat::Proxies::Array
+  end
+
+end

data/lib/treat/proxies/language.rb

@@ -0,0 +1,47 @@
+module Treat::Proxies
+  
+  # This is kind of ugly; need to find a
+  # better solution eventually (?)
+  Treat::Entities::Entity.class_eval do
+    
+    # Rename the true language detection
+    # method to :language_proxied, and
+    # only call it if language detection
+    # is turned on in the configuration.
+    alias :language_proxied :language
+    
+    # Proxy the #language method, defined on
+    # all textual entities, in order to catch
+    # the method call if language detection is
+    # turned off and return the default language
+    # in that case.
+    def language(extractor = nil, options = {})
+  
+      return Treat.core.language.default if
+      !Treat.core.language.detect
+
+      if is_a?(Treat::Entities::Symbol) ||
+        is_a?(Treat::Entities::Number) ||
+        is_a?(Treat::Entities::Punctuation)
+        return Treat.core.language.default
+      end
+
+      dlvl = Treat.core.language.detect_at
+      dklass = Treat::Entities.const_get(dlvl.cc)
+      
+      if self.class.compare_with(dklass) < 1
+        anc = ancestor_with_type(dlvl)
+        return anc.language if anc
+        return self.parent.language if has_parent?
+      end
+
+      extractor ||= Treat.workers.
+      extractors.language.default
+
+      language_proxied(extractor, options)
+
+    end
+
+  end
+
+end

data/lib/treat/proxies/number.rb

@@ -0,0 +1,18 @@
+module Treat::Proxies
+
+  # Install Treat functions on Numeric objects.
+  module Numeric
+    # Include base proxy functionality.
+    include Treat::Proxies::Proxy
+    # Return the entity corresponding to the number.
+    def to_entity(builder = nil)
+      Treat::Entities::Number.from_numeric(self)
+    end
+  end
+  
+  # Include Treat methods on numerics.
+  ::Numeric.class_eval do
+    include Treat::Proxies::Numeric
+  end
+  
+end

data/lib/treat/proxies/proxy.rb

@@ -0,0 +1,25 @@
+# Proxies install builders on core Ruby objects;
+# when a method defined by Treat is called on these
+# objects, the Ruby object is cast to a Treat entity
+# and the method is called on the resultant type.
+module Treat::Proxies
+  
+  # Provides a base functionality for proxies.
+  module Proxy
+    # Build the entity corresponding to the proxied
+    # object and send the method call to the entity.
+    def method_missing(sym, *args, &block)
+      if [:do, :apply].include?(sym) || 
+        Treat::Workers.lookup(sym)
+          to_entity.send(sym, *args)
+      else
+        super(sym, *args, &block)
+      end
+    end
+    # Create an unknown type of entity by default.
+    def to_entity(builder = nil)
+      Treat::Entities::Unknown(self.to_s)
+    end
+  end
+
+end

data/lib/treat/proxies/string.rb

@@ -0,0 +1,18 @@
+module Treat::Proxies
+
+  # Install Treat functions on String objects.
+  module String
+    # Include base proxy functionality.
+    include Treat::Proxies::Proxy
+    # Return the entity corresponding to the string.
+    def to_entity
+      Treat::Entities::Entity.from_string(self)
+    end
+  end
+  
+  # Include Treat methods on strings.
+  ::String.class_eval do
+    include Treat::Proxies::String
+  end
+  
+end

data/lib/treat/version.rb

@@ -1,3 +1,12 @@
 module Treat
-  VERSION = "1.2.0"
+  
+  # The current version of Treat.
+  VERSION = "2.0.0rc1"
+  
+  # Treat requires Ruby >= 1.9.2
+  if RUBY_VERSION < '1.9.2'
+    raise "Treat requires Ruby version 1.9.2 " +
+    "or higher, but current is #{RUBY_VERSION}."
+  end
+
 end

data/lib/treat/{workers.rb → workers/categorizable.rb}

@@ -1,27 +1,27 @@
 # This module creates all the worker categories
 # and the groups within these categories and adds
 # the relevant hooks on the appropriate entities.
-module Treat::Workers
+module Treat::Workers::Categorizable
 
-  require 'treat/workers/group'
+  require_relative 'groupable'
 
   # A lookup table for entity types.
   @@lookup = {}
 
   # Find a worker group based on method.
-  def self.lookup(method)
+  def lookup(method)
     @@lookup[method]
   end
 
-  def self.create_categories
-    Treat.workers.list.each do |cat|
-      create_category(cat.to_s.
+  def categorize!
+    Treat.workers.members.each do |cat|
+      create_category(cat.
       capitalize.intern,
       load_category_conf(cat))
     end
   end
 
-  def self.load_category_conf(name)
+  def load_category_conf(name)
     config = Treat.workers[name]
     if config.nil?
       raise Treat::Exception,
@@ -31,10 +31,11 @@ module Treat::Workers
     config
   end
 
-  def self.create_category(name, conf)
-    category = self.const_set(name, Module.new)
+  def create_category(name, conf)
+    category = Treat::Workers.
+    const_set(name, Module.new)
     conf.each_pair do |group, worker|
-      name = cc(group.to_s).intern
+      name = group.to_s.cc.intern
       category.module_eval do
         @@methods = []; def methods; 
         @@methods; end; def groups; 
@@ -44,7 +45,7 @@ module Treat::Workers
     end
   end
 
-  def self.create_group(name, conf, category)
+  def create_group(name, conf, category)
     group = category.const_set(name, Module.new)
     self.set_group_options(group, conf)
     self.bind_group_targets(group)
@@ -53,17 +54,17 @@ module Treat::Workers
     @@lookup[group.method] = group
   end
 
-  def self.bind_group_targets(group)
+  def bind_group_targets(group)
     group.targets.each do |entity_type|
       entity = Treat::Entities.
-      const_get(cc(entity_type))
+      const_get(entity_type.cc)
       entity.class_eval do
         add_workers group
       end
     end
   end
 
-  def self.register_group_presets(group, conf)
+  def register_group_presets(group, conf)
     return unless conf.respond_to? :presets
     conf.presets.each do |m|
       @@methods << m
@@ -71,9 +72,9 @@ module Treat::Workers
     end
   end
 
-  def self.set_group_options(group, conf)
+  def set_group_options(group, conf)
     group.module_eval do
-      extend Treat::Workers::Group
+      extend Treat::Workers::Groupable
       self.type = conf.type
       self.targets = conf.targets
       if conf.respond_to?(:default)
@@ -90,7 +91,5 @@ module Treat::Workers
       end
     end
   end
-
-  self.create_categories
-
+  
 end

data/lib/treat/workers/extractors/keywords/tf_idf.rb

@@ -1,6 +1,6 @@
-# This retrieves a supplied number of keywords
-# by selecting the N words with the highest TF*IDF
-# for each document.
+# Extracts an arbitrary number of keywords from a
+# document in a collection by selecting its N words
+# with the highest TF*IDF score.
 class Treat::Workers::Extractors::Keywords::TfIdf
   
   # Default options - retrieve 5 keywords.
@@ -8,31 +8,31 @@ class Treat::Workers::Extractors::Keywords::TfIdf
   
   # Annotate a document with an array containing
   # the N words with the highest TF*IDF in that
-  # document,
+  # document.
   def self.keywords(entity, options = {})
     
     options = DefaultOptions.merge(options)
     tf_idfs = {}
     
     entity.each_word do |word|
-      tf_idfs[word] ||= word.tf_idf
+      tf_idf = word.tf_idf
+      if tf_idf
+        tf_idfs[word] ||= tf_idf 
+      end
     end
 
     tf_idfs = tf_idfs.
     sort_by {|k,v| v}.reverse
-
-    if tf_idfs.size <= options[:number]
-      return tf_idfs
-    end
-    
+   
     keywords = []
     i = 0
+    max_count = tf_idfs.size < options[:number] ? tf_idfs.size : options[:number]
     
     tf_idfs.each do |word|
       
       w = word[0].to_s
       next if keywords.include?(w)
-      break if i > options[:number]
+      break if i > max_count
       keywords << w
       
       i += 1

data/lib/treat/workers/extractors/language/what_language.rb

@@ -1,9 +1,11 @@
-# Adaptor for the 'whatlanguage' gem, which
-# performs probabilistic language detection.
-# The library works by checking for the presence
-# of words with bloom filters built from
-# dictionaries based upon each source language.
-module Treat::Workers::Extractors::Language::WhatLanguage
+# Language detection using a probabilistic algorithm
+# that checks for the presence of words with Bloom 
+# filters built from dictionaries for each language.
+#
+# Original paper: Grothoff. 2007. A Quick Introduction to 
+# Bloom Filters. Department of Computer Sciences, Purdue 
+# University.
+class Treat::Workers::Extractors::Language::WhatLanguage
 
   # Require the 'whatlanguage' gem.
   silence_warnings { require 'whatlanguage'  }