treat 1.2.0 → 2.0.0rc1

data/lib/treat/entities/{abilities → entity}/exportable.rb

@@ -1,7 +1,7 @@
-module Treat::Entities::Abilities::Exportable
+module Treat::Entities::Entity::Exportable
   
   def export(problem)
-    ds = Treat::Core::DataSet.new(problem)
+    ds = Treat::Learning::DataSet.new(problem)
     each_entity(problem.question.target) do |e|
       ds << e
     end

data/lib/treat/entities/{abilities → entity}/iterable.rb

@@ -1,4 +1,4 @@
-module Treat::Entities::Abilities::Iterable
+module Treat::Entities::Entity::Iterable
 
   # Yields each entity of any of the supplied
   # types in the children tree of this Entity.
@@ -6,12 +6,12 @@ module Treat::Entities::Abilities::Iterable
   # #each. It does not yield the top element being
   # recursed.
   #
-  # This function NEEDS to be ported to C.
+  # This function NEEDS to be ported to C. #FIXME
   def each_entity(*types)
     types = [:entity] if types.size == 0
     f = false
     types.each do |t2|
-      if is_a?(Treat::Entities.const_get(cc(t2)))
+      if is_a?(Treat::Entities.const_get(t2.cc))
         f = true; break
       end
     end
@@ -57,7 +57,7 @@ module Treat::Entities::Abilities::Iterable
   def ancestor_with_type(type)
     return unless has_parent?
     ancestor = @parent
-    type_klass = Treat::Entities.const_get(cc(type))
+    type_klass = Treat::Entities.const_get(type.cc)
     while not ancestor.is_a?(type_klass)
       return nil unless (ancestor && ancestor.has_parent?)
       ancestor = ancestor.parent
@@ -105,18 +105,6 @@ module Treat::Entities::Abilities::Iterable
     end
     i
   end
-  
-  # Return the first element in the array, warning if not
-  # the only one in the array. Used for magic methods: e.g.,
-  # the magic method "word" if called on a sentence with many 
-  # words, Treat will return the first word, but warn the user.
-  def first_but_warn(array, type)
-    if array.size > 1
-      warn "Warning: requested one #{type}, but" +
-      " there are many #{type}s in this entity."
-    end
-    array[0]
-  end
 
   
 end

data/lib/treat/entities/{abilities → entity}/magical.rb

@@ -1,27 +1,20 @@
-module Treat::Entities::Abilities::Magical
+module Treat::Entities::Entity::Magical
 
   # Parse "magic methods", which allow the following
   # syntaxes to be used (where 'word' can be replaced
   # by any entity type, e.g. token, zone, etc.):
   #
-  # - each_word : iterate over each entity of type word.
-  # - words: return an array of words in the entity.
+  # - each_word : iterate over each children of type word.
+  # - words: return an array of children words.
   # - word: return the first word in the entity.
   # - word_count: return the number of words in the entity.
-  # - words_with_*(value) (where  is an arbitrary feature):
-  #   return the words that have the given feature.
-  # - word_with_*(value) : return the first word with
-  #   the feature specified by * in value.
-  #
-  # Also provides magical methods for types of words:
-  #
-  # - each_noun:
-  # - nouns:
-  # - noun:
-  # - noun_count:
-  # - nouns_with_*(value)
-  # - noun_with_*(value)
+  # - words_with_*(value) (where * is an arbitrary feature):
+  #   return the words that have the given feature set to value.
   #
+  # Also provides magical methods for types of words (each_noun,
+  # nouns, noun_count, nouns_with_*(value) noun_with_*(value), etc.)
+  # For this to be used, the words in the text must have been 
+  # tokenized and categorized in the first place.
   def magic(sym, *args)
 
     # Cache this for performance.
@@ -80,9 +73,21 @@ module Treat::Entities::Abilities::Magical
     elsif method =~ /^frequency_in_#{@@entities_regexp}$/
       frequency_in($1.intern)
     else
-      return :no_magic
+      return :no_magic # :-(
     end
+    
   end
   
+  # Return the first element in the array, warning if not
+  # the only one in the array. Used for magic methods: e.g.,
+  # the magic method "word" if called on a sentence with many 
+  # words, Treat will return the first word, but warn the user.
+  def first_but_warn(array, type)
+    if array.size > 1
+      warn "Warning: requested one #{type}, but" +
+      " there are many #{type}s in this entity."
+    end
+    array[0]
+  end
   
 end

data/lib/treat/entities/entity/registrable.rb

@@ -0,0 +1,36 @@
+# Registers the entities ocurring in the subtree of
+# a node as children are added. Also registers text
+# occurrences for word groups and tokens (n grams).
+module Treat::Entities::Entity::Registrable
+
+  # Registers a token or phrase in the registry.
+  # The registry keeps track of children by id,
+  # by entity type, and also keeps the position
+  # of the entity in its parent entity.
+  def register(entity)
+    unless @registry 
+      @count, @registry = 0, 
+      {id: {}, value: {}, position:{}, type: {}} 
+    end
+    if entity.is_a?(Treat::Entities::Token) ||
+      entity.is_a?(Treat::Entities::Group)
+      val = entity.to_s.downcase
+      @registry[:value][val] ||= 0
+      @registry[:value][val] += 1
+    end
+    @registry[:id][entity.id] = true
+    @registry[:type][entity.type] ||= 0
+    @registry[:type][entity.type] += 1
+    @registry[:position][entity.id] = @count
+    @count += 1
+    @parent.register(entity) if has_parent?
+  end
+
+  # Backtrack up the tree to find a token registry,
+  # by default the one in the root node of the tree.
+  def registry(type = nil)
+    (has_parent? && type != self.type) ?
+    @parent.registry(type) : @registry
+  end
+
+end

data/lib/treat/entities/{abilities → entity}/stringable.rb

@@ -1,18 +1,22 @@
 # Gives entities the ability to be converted
 # to string representations (#to_string, #to_s,
 # #to_str, #inspect, #print_tree).
-module Treat::Entities::Abilities::Stringable
-
-  # Return the entity's true string value in
-  # plain text format. Non-terminal entities
-  # will normally have an empty value.
-  def to_string; @value; end
-
+module Treat::Entities::Entity::Stringable
+  
+  # Returns the entity's true string value.
+  def to_string;  @value.dup; end
+  
+  # Returns an array of the childrens' string
+  # values, found by calling #to_s on them.
+  def to_a; @children.map { |c| c.to_s }; end
+  
+  alias :to_ary :to_a
+  
   # Returns the entity's string value by
   # imploding the value of all terminal
   # entities in the subtree of that entity.
   def to_s
-    @value != '' ? @value : implode.strip
+    has_children? ? implode.strip : @value.dup
   end
   
   # #to_str is the same as #to_s.
@@ -24,12 +28,10 @@ module Treat::Entities::Abilities::Stringable
   def short_value(max_length = 30)
     s = to_s
     words = s.split(' ')
-    if s.length < max_length
-      s
-    else
-      words[0..2].join(' ') + ' [...] ' +
-      words[-2..-1].join(' ')
-    end
+    return s if (s.length < max_length) ||
+    !(words[0..2] && words[-2..-1])
+    words[0..2].join(' ') + ' [...] ' +
+    words[-2..-1].join(' ')
   end
 
   # Print out an ASCII representation of the tree.
@@ -38,7 +40,8 @@ module Treat::Entities::Abilities::Stringable
   # Return an informative string representation
    # of the entity.
    def inspect
-     s = "#{cl(self.class)} (#{@id.to_s})"
+     name = self.class.mn
+     s = "#{name} (#{@id.to_s})"
      if caller_method(2) == :inspect
        @id.to_s
      else

data/lib/treat/entities/entity.rb

@@ -1,94 +1,106 @@
 module Treat::Entities
 
-  module Abilities; end
-
-  # Require abilities.
-  p = Treat.paths.lib +
-  'treat/entities/abilities/*.rb'
-  Dir.glob(p).each { |f| require f }
-
+  # Basic tree structure.
   require 'birch'
-
+  
+  # The Entity class extends a basic tree structure 
+  # (written in C for optimal speed) and represents 
+  # any form of textual entityin a processing task 
+  # (this could be a collection of documents, a 
+  # single document, a single paragraph, etc.)
+  # 
+  # Classes that extend Entity provide the concrete
+  # behavior corresponding to the relevant entity type. 
+  # See entities.rb for a full list and description of 
+  # the different entity types in the document model.
   class Entity < ::Birch::Tree
 
-    # A Symbol representing the lowercase
-    # version of the class name.
+    # A symbol representing the lowercase
+    # version of the class name. This is 
+    # the only attribute that the Entity
+    # class adds to the Birch::Tree class.
     attr_accessor :type
+    
+    # Autoload all the classes in /abilities.
+    path = File.expand_path(__FILE__)
+    patt = File.dirname(path) + '/entity/*.rb'
+    Dir.glob(patt).each { |f| require f }
+    
+    # Implements support for #register, #registry.
+    include Registrable
 
-    # Implements support for #register,
-    # #registry, and #contains_* methods.
-    include Abilities::Registrable
+    # Implement support for #self.call_worker, etc.
+    extend Delegatable
 
-    # Implement support for #self.add_workers
-    extend Abilities::Delegatable
+    # Implement support for #self.print_debug, etc.
+    extend Debuggable
 
-    # Implement support for #self.print_debug and
-    # #self.invalid_call_msg
-    extend Abilities::Debuggable
+    # Implement support for #self.build and #self.from_*
+    extend Buildable
 
-    # Implement support for #self.build
-    # and #self.from_*
-    extend Abilities::Buildable
+    # Implement support for #apply (previously #do).
+    include Applicable
 
-    # Implement support for #do.
-    include Abilities::Doable
+    # Implement support for #frequency, #frequency_in,
+    # #frequency_of, #position, #position_from_end, etc.
+    include Countable
 
-    # Implement support for #frequency,
-    # #frequency_in_parent and #position_in_parent.
-    include Abilities::Countable
-
-    # Implement support for #magic.
-    include Abilities::Magical
+    # Implement support for over 100 #magic methods!
+    include Magical
 
     # Implement support for #to_s, #inspect, etc.
-    include Abilities::Stringable
+    include Stringable
 
-    # Implement support for #check_has
-    # and #check_hasnt_children?
-    include Abilities::Checkable
+    # Implement support for #check_has and others.
+    include Checkable
 
     # Implement support for #each_entity, as well as
     # #entities_with_type, #ancestors_with_type,
-    # #entities_with_feature, #entities_with_category.
-    include Abilities::Iterable
-
-    # Implement support for #export to export
-    # a line of a data set based on a classification.
-    include Abilities::Exportable
+    # #entities_with_feature, #entities_with_category, etc.
+    include Iterable
 
-    # Implement support for #copy_into.
-    include Abilities::Copyable
+    # Implement support for #export, allowing to export 
+    # a data set row from the receiving entity.
+    include Exportable
 
     # Implement support for #self.compare_with
-    extend Abilities::Comparable
+    extend Comparable
 
     # Initialize the entity with its value and
     # (optionally) a unique identifier. By default,
     # the object_id will be used as id.
     def initialize(value = '', id = nil)
-      id ||= object_id
-      super(value, id)
+      id ||= object_id; super(value, id)
       @type = :entity if self == Entity
-      @type ||= ucc(cl(self.class)).intern
+      @type ||= self.class.mn.ucc.intern
     end
 
     # Add an entity to the current entity.
     # Registers the entity in the root node
     # token registry if the entity is a leaf.
-    #
-    # @see Treat::Registrable
+    # Unsets the parent node's value; in order
+    # to keep the tree clean, only the leaf
+    # values are stored.
+    # 
+    # Takes in a single entity or an array of 
+    # entities. Returns the first child supplied.
+    # If a string is 
     def <<(entities, clear_parent = true)
-      unless entities.is_a? Array
-        entities = [entities]
-      end
-      entities.each do |entity|
-        register(entity)
-      end
+      entities = (entities.is_a?(::String) ||
+      entities.is_a?(::Numeric)) ? 
+      entities.to_entity : entities
+      entities = entities.is_a?(::Array) ?
+      entities : [entities]
+      # Register each entity in this node.
+      entities.each { |e| register(e) }
+      # Pass to the <<() method in Birch.
       super(entities)
+      # Unset the parent value if necessary.
       @parent.value = '' if has_parent?
-      entities[0]
+      # Return the first child.
+      return entities[0]
     end
-
+    
     # Catch missing methods to support method-like
     # access to features (e.g. entity.category
     # instead of entity.features[:category]) and to
@@ -102,29 +114,26 @@ module Treat::Entities
     # sugar for the #self.build method.
     def method_missing(sym, *args, &block)
       return self.build(*args) if sym == nil
-
-      if !@features.has_key?(sym)
-        r = magic(sym, *args, &block)
-        return r unless r == :no_magic
-        begin
-          super(sym, *args, &block)
-        rescue NoMethodError
-          raise Treat::Exception,
-          if Treat::Workers.lookup(sym)
-            msg = "Method #{sym} cannot " +
-            "be called on a #{type}."
-          else
-            msg = "Method #{sym} does not exist."
-            msg += did_you_mean?(
-            Treat::Workers.methods, sym)
-          end
-        end
-      else
-        @features[sym]
-      end
-
+      return @features[sym] if @features.has_key?(sym)
+      result = magic(sym, *args, &block)
+      return result unless result == :no_magic
+      begin; super(sym, *args, &block)
+      rescue NoMethodError; invalid_call(sym); end
     end
-
+    
+    # Raises a Treat::Exception saying that the 
+    # method called was invalid, and that the 
+    # requested method does not exist. Also
+    # provides suggestions for misspellings.
+    def invalid_call(sym)
+      msg = Treat::Workers.lookup(sym) ?
+      "Method #{sym} can't be called on a #{type}." :
+      "Method #{sym} is not defined by Treat." +
+      Treat::Helpers::Help.did_you_mean?(
+      Treat::Workers.methods, sym)
+      raise Treat::Exception, msg
+    end
+    
   end
 
 end

data/lib/treat/exception.rb

@@ -0,0 +1,3 @@
+# Custom exception class allowing to differentiate
+# Treat errors from library/Ruby exceptions.
+class Treat::Exception < ::Exception; end

data/lib/treat/helpers/hash.rb

@@ -0,0 +1,29 @@
+# Helper methods to manipulate hashes.
+class Treat::Helpers::Hash
+
+  # Mixin to allow conversion of hashes to
+  # nested structs with the keys as attributes.
+  module ToStruct
+    # Converts a hash to nested structs.
+    def to_struct
+      hash = self
+      symbols = hash.keys.select { |k| 
+      !k.is_a?(Symbol) }.size
+      return hash if symbols > 0
+      klass = Struct.new(*hash.keys)
+      struct = klass.new(*hash.values)
+      hash.each do |key, value|
+        if value.is_a?(Hash)
+          v = value.to_struct
+          struct[key] = v
+        end
+      end; return struct
+    end
+  end
+  
+  # Include the mixins on the core Hash class.
+  Hash.class_eval do
+    include Treat::Helpers::Hash::ToStruct
+  end
+
+end

data/lib/treat/helpers/help.rb

@@ -0,0 +1,35 @@
+# Helper methods to detect misspellings
+# and suggest alternatives to the user.
+class Treat::Helpers::Help
+  
+  # Search the list to see if there are
+  # words similar to #name in the #list
+  # If yes, return a string saying
+  # "Did you mean ... ?" with the names.
+  def self.did_you_mean?(list, name)
+    return '' # Fix
+    list = list.map { |e| e.to_s }
+    name = name.to_s
+    sugg = []
+    list.each do |element|
+      l = self.levenshtein(element,name)
+      if  l > 0 && l < 2
+        sugg << element
+      end
+    end
+    unless sugg.size == 0
+      if sugg.size == 1
+        msg += " Perhaps you meant '#{sugg[0]}' ?"
+      else
+        sugg_quote = sugg[0..-2].map do
+          |x| '\'' + x + '\''
+        end
+        msg += " Perhaps you meant " +
+        "#{sugg_quote.join(', ')}," +
+        " or '#{sugg[-1]}' ?"
+      end
+    end
+    msg
+  end
+  
+end

data/lib/treat/helpers/object.rb

@@ -0,0 +1,55 @@
+# Methods related to object reflection.
+class Treat::Helpers::Object
+  
+  # Allow introspection onto what method called
+  # another one at runtime (useful for debugging).
+  module CallerMethod
+    # Pattern to match method from trace.
+    CMPattern = /^(.+?):(\d+)(?::in `(.*)')?/
+    # Return the name of the method that 
+    # called the method that calls this method.
+    def caller_method(n = 3)
+      at = caller(n).first
+      CMPattern =~ at
+      Regexp.last_match[3].
+      gsub('block in ', '').intern
+    end
+  end
+
+  # Retrieve the last name of a class/module
+  # (i.e. the part after the last "::").
+  module ModuleName
+    def module_name; self.to_s.split('::')[-1]; end
+    alias :mn :module_name
+  end
+  
+  module Verbosity
+    # Runs a block of code without warnings.
+    def silence_warnings(&block)
+      warn_level = $VERBOSE; $VERBOSE = nil
+      result = block.call; $VERBOSE = warn_level
+      result
+    end
+    # Runs a block of code while blocking stdout.
+    def silence_stdout(log = '/dev/null')
+      unless Treat.core.verbosity.silence
+        yield; return
+      end
+      file, old, ret = File.new(log, 'w'), 
+      $stdout.dup, nil; $stdout.reopen(file)
+      ret = yield; $stdout = old; return ret
+    end
+  end
+  
+  # Allow getting the caller method in any context.
+  Object.class_eval do
+    include Treat::Helpers::Object::CallerMethod
+    include Treat::Helpers::Object::Verbosity
+  end
+  
+  # Allow getting the last name of any module/class.
+  Module.class_eval do
+    include Treat::Helpers::Object::ModuleName
+  end
+  
+end

data/lib/treat/helpers/string.rb

@@ -0,0 +1,124 @@
+# Helper methods for camel casing and 
+# escaping standard strings and symbols.
+class Treat::Helpers::String
+
+  # Utility to escape floating point numbers
+  # from strings (useful for a variety of 
+  # applications, including chunking, segmenting
+  # and tokenizing, to exclude periods that 
+  # are not sentence terminators).
+  module Escapable
+    
+    # Escape char to use.
+    EscapeChar = '^^^'
+    # Regex for escape.
+    Regex = /([0-9]+)\.([0-9]+)/
+    
+    # Escape float periods with EscapeChar.
+    def escape_floats!
+      to_s.gsub!(Regex) { $1 + EscapeChar + $2 }
+    end
+    
+  end
+
+  # Counterpart to Treat::Helpers::Escapable;
+  # unescapes floats, restoring the orgiinal text.
+  module Unescapable
+    
+    # Escaped for regex.
+    EscapedEscapeChar = '\^\^\^'
+    # Regex for unescape.
+    Regex = /([0-9]+)#{EscapedEscapeChar}([0-9]+)/
+    
+    # Unescape float periods (restore text).
+    def unescape_floats!
+      to_s.gsub!(Regex) { $1 + '.' + $2 }
+    end
+    
+  end
+  
+  # Transform an un_camel_cased string
+  # into a CamelCased string. This is
+  # available on String and Symbol.
+  module CamelCaseable
+
+    # A cache to optimize camel casing.
+    @@cc_cache = {}
+    
+    # Regex for camel casing.
+    Regex = /^[a-z]|_[a-z]/
+    
+    # Convert un_camel_case to CamelCase.
+    def camel_case
+      o_phrase, phrase = to_s, to_s.dup
+      if @@cc_cache[o_phrase]
+        return @@cc_cache[o_phrase] 
+      end
+      if Treat.core.acronyms.include?(phrase)
+        phrase = phrase.upcase
+      else
+        phrase.gsub!(Regex) { |a| a.upcase }
+        phrase.gsub!('_', '')
+      end
+      @@cc_cache[o_phrase] = phrase
+    end
+
+    alias :cc :camel_case
+
+  end
+
+  # Counterpart of Treat::Helpers::CamelCaseable;
+  # transforms a CamelCase string to its un_camel_
+  # case corresponding form.
+  module UnCamelCaseable
+    
+    # A cache to optimize un camel casing.
+    @@ucc_cache = {}
+
+    # Convert CamelCase to un_camel_case.
+    def un_camel_case
+      o_phrase, phrase = to_s, to_s.dup
+      if @@ucc_cache[o_phrase]
+        return @@ucc_cache[o_phrase] 
+      end
+      acros = Treat.core.acronyms
+      if !acros.include?(phrase.downcase)
+        phrase.gsub!(/[A-Z]/) do |p| 
+          '_' + p.downcase
+        end
+        if phrase[0] == '_'
+          return phrase = phrase[1..-1] 
+        end
+      else
+        phrase = phrase.downcase
+      end
+      @@ucc_cache[o_phrase] = phrase
+    end
+
+    alias :ucc :un_camel_case
+  
+  end
+  
+  # Determines whether module is 
+  # an "-able" mixin kind of thing.
+  module IsMixin
+    def is_mixin?; to_s[-4..-1] == 'able'; end
+  end
+  
+  # Graft the helpers onto the string module.
+  String.class_eval do
+    include Treat::Helpers::String::CamelCaseable
+    include Treat::Helpers::String::UnCamelCaseable
+    include Treat::Helpers::String::Escapable
+    include Treat::Helpers::String::Unescapable
+    include Treat::Helpers::String::IsMixin
+  end
+  
+  # Graft camel casing onto symbols.
+  Symbol.class_eval do
+    include Treat::Helpers::String::CamelCaseable
+    include Treat::Helpers::String::UnCamelCaseable
+  end
+  
+  
+end