treat 0.1.1 → 0.1.2

data/INSTALL

@@ -0,0 +1 @@
+See the Wiki: https://github.com/louismullie/treat/wiki/Installing-Treat

data/README

@@ -0,0 +1,3 @@
+Treat - Text Retrieval and Annotation Toolkit
+
+See the wiki for more information at https://github.com/louismullie/treat/wiki/.

data/TODO

@@ -1,23 +1,29 @@
-== Before first release
+## Urgent
 
-- Linkers, inflectors
+- Linkers
 - Check taggers for context
 - Stanford dependencies parse
-- Enju : make sure Enju parses constituents, remove_last, abbr
+- Enju: test
 - Ocropus => use better function
-- Optimize magic methods... bigger or smaller, is_token? type methods, phrase categories.
+- Optimize magic methods... is_token? type methods, phrase categories.
 - Move statistics?
 - Synset class move
 - general procedure for options, check that user doesn't want to change options...
-- Resources: dependencies vs. edges, PTB function tags
+- Languages: dependencies vs. edges, PTB function tags
 - Check for # Fix everywhere
 - Check paths; parse bin paths
 - Ferret, Natural Inputs
 - Use consistently delegate
 - Text becomes section
+- Remove top level
+- Loading multiple JARs
+- Tokenized sentences are not parsed
+- Documentation
+- Remove feature
 
-=== Eventually
+## Eventually
 
+- English inflector
 - RDF output
 - Apache OpenNLP
 - Ariel
@@ -44,24 +50,6 @@
 - Probabilistic features: rchardet19, what_language
 - Enju multithreading ?
 - String type detector for other languages
-
 - Automatic benchmark
-
-=== Distant future
-
-- Spell Cheker Raspell
-- Multithreading
-
-=== Checklist before releasing
-
-- Remove code from main page
-- Remove lib path from tests and main page
-- Remove docs
-- gem19 build treat.gemspec
-
-=== Performance
-
-- Cache results that get computed often
-- Use .size == 0 instead of .empty?
-- Optimize method_missing using define_method, even dynamically
-- Array include is slow
+- Raspell spell checker
+- Multithreading

data/bin/INFO

@@ -1 +1 @@
-This is where you can put your JAR files.
+This is where Treat will look for the Stanford JAR files by default. You can change this to another directory by setting Treat.bin = '/path/to/your/folder/' at runtime.

data/lib/treat/buildable.rb

@@ -1,6 +1,9 @@
 module Treat
+  # Represents an object that can be built
+  # from a folder of files, a specific file,
+  # a string or a numeric object. This class
+  # is pretty much self-explanatory.
   module Buildable
-
     def from_anything(file_or_value, id)
       if File.readable?(file_or_value.to_s)
         from_file(file_or_value)
@@ -9,11 +12,11 @@ module Treat
       elsif file_or_value.is_a?(Numeric)
         from_numeric(file_or_value)
       else
-        raise "Unrecognizable input #{file_or_value}. "+
+        raise Treat::Exception,
+        "Unrecognizable input #{file_or_value}. "+
         "Use filename, folder, text or a number."
       end
     end
-    
     def from_string(string)
       if self == Treat::Entities::Document ||
         self == Treat::Entities::Collection
@@ -37,7 +40,6 @@ module Treat
       end
       return Treat::Entities::Unknown.new(string)
     end
-
     def from_numeric(numeric)
       unless self == Treat::Entities::Number
         raise Treat::Exception,
@@ -46,7 +48,6 @@ module Treat
       end
       Treat::Entities::Number.new(numeric.to_s)
     end
-
     def from_folder(folder)
       unless FileTest.directory?(folder)
         raise Treat::Exception,
@@ -69,7 +70,6 @@ module Treat
       end
       c
     end
-
     def from_file(file)
       unless File.readable?(file)
         raise Treat::Exception,
@@ -79,7 +79,8 @@ module Treat
         from_folder(file)
       else
         ext = file.split('.')[-1]
-        ext = 'yaml' if ext == 'yml' # Humanize the yml extension.
+        # Humanize the yaml extension.
+        ext = 'yaml' if ext == 'yml'
         if Treat::Formatters::Unserializers.list.
           include?(ext.downcase.intern)
           from_serialized_file(file)
@@ -88,7 +89,6 @@ module Treat
         end
       end
     end
-
     def from_raw_file(file)
       unless self == Treat::Entities::Document
         raise Treat::Exception,
@@ -98,9 +98,9 @@ module Treat
       d = Treat::Entities::Document.new(file)
       d.read
     end
-
     def from_serialized_file(file)
-      unless [Treat::Entities::Document, Treat::Entities::Collection].include?(self)
+      unless [Treat::Entities::Document, 
+        Treat::Entities::Collection].include?(self)
         raise Treat::Exception,
         "Cannot create something else than a " +
         "document from raw file '#{file}'."
@@ -110,6 +110,5 @@ module Treat
       d.children[0].set_as_root!
       d.children[0]
     end
-
   end
 end

data/lib/treat/categories.rb

@@ -1,16 +1,18 @@
 module Treat
+  # This module keeps track of all categories that
+  # exist and the methods they implement, and is
+  # responsible for including the categories.
   module Categories
-    # Modify the module that includes Category to
-    # setup autoload, delegators and provide a list
-    # of methods.
+    # A list of categories.
     class << self; attr_accessor :list; end
     self.list = []
-    # Boolean - does any of the categories
-    # groups respond to the symbol.
+    # Boolean - does any of the categories have 
+    # a method that corresponds to sym?
     def self.have_method?(sym); methods.include?(sym); end
+    # Cache the list of methods once it has been computed.
+    @@methods = []
     # Provide a list of all methods implemented
     # by all Treat categories.
-    @@methods = []
     def self.methods
       return @@methods unless @@methods.empty?
       self.list.each do |ns|

data/lib/treat/category.rb

@@ -1,7 +1,11 @@
 module Treat
-  # Provides functions common to all algorithm categories.
+  # Clusters together groups of algorithms that
+  # perform similar functions.
   module Category
+    # Require the Group class.
     require 'treat/group'
+    # Add delegators to the Entities based on the
+    # configuration for a given category.
     def self.extended(category)
       Treat::Categories.list << category
       category.module_eval do
@@ -14,9 +18,10 @@ module Treat
         end
       end
     end
+    # Provides a list of groups within this category.
     def groups; self.constants; end
     # Provide a list of methods implemented in
-    # the groups contained within that
+    # the groups contained within this category.
     def methods
       methods = []
       groups.each do |group|

data/lib/treat/delegatable.rb

@@ -2,34 +2,8 @@ module Treat
   # Makes a class delegatable, allowing calls on it to be forwarded
   # to a delegate class performing the appropriate call.
   module Delegatable
-
-    # Get the default delegate for that language
-    # inside the given group.
-    def get_language_delegate(language, group)
-      lang = Treat::Resources::Languages.describe(language)
-      lclass = cc(lang).intern
-      if Treat::Resources::Delegates.constants.include?(lclass)
-        cat = group.to_s.split('::')[-2].intern
-        lclass = Treat::Resources::Delegates.
-        const_get(lclass).const_get(cat)
-        g = ucc(cl(group)).intern
-        if !lclass[g] || !lclass[g][0]
-          d = ucc(cl(group))
-          d.gsub!('_', ' ')
-          d = d[0..-2] if d[-1] == 's'
-          d = 'delegator to find ' + d
-          raise Treat::Exception, "No #{d}" +
-          " is available for the #{lang} language."
-        end
-        return lclass[g][0]
-      else
-        raise Treat::Exception,
-        "Language '#{lang}' is not supported (yet)."
-      end
-    end
-
     # Add decorator methods to entities.
-    def decorate(group, m)
+    def add_decorators(group, m)
       decorators = group.methods -
       Object.methods -
       [:type, :type=, :targets, :targets=,
@@ -42,49 +16,83 @@ module Treat
         end
       end
     end
-
-    # Raise an exception and suggest alternatives.
-    def delegate_not_found(klass, group)
-      "Algorithm '#{ucc(klass)}' couldn't be found in group #{group}." +
-      did_you_mean?(group.list.map { |c| ucc(c) }, ucc(klass))
-    end
-
     # Add delegator group to all entities of a class.
     def add_delegators(group)
       # Define each method in group.
       self.class_eval do
         m = group.method
-        decorate(group, m)
+        add_decorators(group, m)
         define_method(m) do |delegate=nil, options={}|
           decorator = options.delete(:decorator)
           puts self.id if !@features
           if !@features[m].nil?
             @features[m]
           else
-            if delegate.nil?
-              delegate = group.default.nil? ? 
-              self.class.get_language_delegate(language, group) :
-              group.default
-              raise "No default delegate for #{group}." if delegate == :none
-            end
-            if not group.list.include?(delegate)
-              raise Treat::Exception,
-              self.class.delegate_not_found(delegate, group)
-            else
-              delegate_klass = group.const_get(:"#{cc(delegate.to_s)}")
-              result = accept(group, delegate_klass, m, options)
-              if decorator
-                result = group.send(decorator, self, result)
-              end
-              if group.type == :annotator
-                f = decorator.nil? ? m : decorator
-                @features[f] = result
-              end
-              result
-            end
+            self.class.call_delegator(
+            self, m, delegate, decorator, 
+            group, options)
           end
         end
       end
     end
+    # Call a delegator.
+    def call_delegator(entity, m, delegate, decorator, group, options)
+      if delegate.nil?
+        delegate = get_missing_delegate(entity, group)
+      end
+      if not group.list.include?(delegate)
+        raise Treat::Exception, delegate_not_found(delegate, group)
+      else
+        delegate_klass = group.const_get(:"#{cc(delegate.to_s)}")
+        result = entity.accept(group, delegate_klass, m, options)
+        if decorator
+          result = group.send(decorator, self, result)
+        end
+        if group.type == :annotator
+          f = decorator.nil? ? m : decorator
+          entity.features[f] = result
+        end
+        result
+      end
+    end
+    # Get the default delegate for that language
+    # inside the given group.
+    def get_language_delegate(language, group)
+      lang = Treat::Languages.describe(language)
+      lclass = cc(lang).intern
+      if Treat::Languages.constants.include?(lclass)
+        cat = group.to_s.split('::')[-2].intern
+        lclass = Treat::Languages.const_get(lclass).const_get(cat)
+        g = ucc(cl(group)).intern
+        if !lclass[g] || !lclass[g][0]
+          d = ucc(cl(group))
+          d.gsub!('_', ' ')
+          d = d[0..-2] if d[-1] == 's'
+          d = 'delegator to find ' + d
+          raise Treat::Exception, "No #{d}" +
+          " is available for the #{lang} language."
+        end
+        return lclass[g][0]
+      else
+        raise Treat::Exception,
+        "Language '#{lang}' is not supported (yet)."
+      end
+    end
+    # Get which delegate to use if none has been supplied.
+    def get_missing_delegate(entity, group)
+      delegate = group.default.nil? ?
+      self.get_language_delegate(entity.language, group) :
+      group.default
+      if delegate == :none
+        raise NAT::Exception,
+        "There is intentionally no default delegate for #{group}."
+      end
+      delegate
+    end
+    # Return an error message and suggest possible typos.
+    def delegate_not_found(klass, group)
+      "Algorithm '#{ucc(klass)}' couldn't be found in group #{group}." +
+      did_you_mean?(group.list.map { |c| ucc(c) }, ucc(klass))
+    end
   end
 end

data/lib/treat/detectors/encoding/r_chardet19.rb

@@ -2,7 +2,7 @@ module Treat
   module Detectors
     module Encoding
       # Require the 'rchardet19' gem.
-      silently { require 'rchardet19' }
+      silence_warnings { require 'rchardet19' }
       # A wrapper for the 'rchardet19' gem, which
       # detects the encoding of a file.
       class RChardet19

data/lib/treat/detectors/language/language_detector.rb

@@ -9,7 +9,8 @@ module Treat
             dlvl = Treat.language_detection_level
             if (Entities.rank(entity.type) < Entities.rank(dlvl)) &&
                entity.has_parent?
-                return entity.ancestor_with_type(dlvl).language
+               anc = entity.ancestor_with_type(dlvl)
+               return anc.language if anc
             end
           end
         end

data/lib/treat/detectors/language/what_language.rb

@@ -2,7 +2,7 @@ module Treat
   module Detectors
     module Language
       # Require the 'whatlanguage' gem.
-      silently { require 'whatlanguage'  }
+      silence_warnings { require 'whatlanguage'  }
       # Adaptor for the 'whatlanguage' gem, which
       # performs probabilistic language detection.
       class WhatLanguage < LanguageDetector
@@ -19,7 +19,7 @@ module Treat
           all = @@wl.process_text(entity.to_s)
           lang = {}
           all.each do |k,v|
-            lang[Treat::Resources::Languages.find(k)] = v
+            lang[Treat::Languages.find(k)] = v
           end
           Treat::Feature.new(lang).best
         end

data/lib/treat/detectors.rb

@@ -1,6 +1,9 @@
 module Treat
   # Detectors detect a specific meta-information about
   # an entity, such as encoding, format and language.
+  # 
+  # Detectors are language-independent, and thus they 
+  # are default algorithms specified for each of them.
   module Detectors
     # Group for algorithms that detect encoding.
     module Encoding 

data/lib/treat/entities/entity.rb

@@ -93,7 +93,7 @@ module Treat
       # dispatches done by Ruby to improve performance.
       def parse_magic_method(sym, *args, &block)
         @@entities_regexp ||= "(#{Treat::Entities.list.join('|')})"
-        @@cats_regexp ||= "(#{Treat::Resources::Categories::List.join('|')})"
+        @@cats_regexp ||= "(#{Treat::Languages::English::Categories.join('|')})"
         method = sym.to_s =~ /entities/ ?
         sym.to_s.gsub('entities', 'entitys'):
         method = sym.to_s

data/lib/treat/entities.rb

@@ -4,15 +4,14 @@ module Treat
   #
   # - Collection
   # - Document
-  # - Text
   # - Zone (a Section, Title, Paragraph, or List)
   # - Sentence
   # - Constituent (a Phrase or Clause)
   # - Token (a Word, Number, Punctuation, or Symbol).
   module Entities
-    # Require Entity first, since the other classes
-    # extend this class.
+    # Require Entity first.
     require 'treat/entities/entity'
+    # Then require all possible entities.
     require 'treat/entities/collection'
     require 'treat/entities/document'
     require 'treat/entities/text'
@@ -40,13 +39,13 @@ module Treat
     # comparison of entity types.
     def self.rank(type)
       klass = Entities.const_get(cc(type))
-      return 6 if klass == Collection || klass < Collection
-      return 5 if klass == Document || klass < Document
-      return 4 if klass == Text || klass < Text
-      return 3 if klass == Zone || klass < Zone
-      return 2 if klass == Sentence || klass < Sentence
-      return 1 if klass == Constituent || klass < Constituent
-      return 0 if klass == Token || klass < Token
+      compare = lambda { |a,b| a == b || a < b }
+      return 0 if compare.call(klass, Token)
+      return 1 if compare.call(klass, Constituent)
+      return 2 if compare.call(klass, Sentence)
+      return 4 if compare.call(klass, Document)
+      return 3 if compare.call(klass, Section)
+      return 5 if compare.call(klass, Collection)
     end
   end
 end

data/lib/treat/exception.rb

@@ -1,5 +1,7 @@
 module Treat
-  # Exception class for the Treat library.
+  # Custom exception class for the Treat toolkit.
+  # Used to distinguish between errors raised by
+  # gems or Ruby from errors raised by the toolkit.
   class Exception < ::Exception
   end
 end

data/lib/treat/extractors/named_entity/abner.rb

@@ -3,7 +3,7 @@ module Treat
     module NamedEntity
       class Abner
         # Require the Ruby-Java bridge.
-        silently do
+        silence_warnings do
           require 'rjb'
           Rjb::load('', ['-Xms256M', '-Xmx512M'])
           puts Rjb.import('tagger')

data/lib/treat/extractors/named_entity/stanford.rb

@@ -3,7 +3,7 @@ module Treat
     module NamedEntity
       class Stanford
         # Require the Ruby-Java bridge.
-        silently do 
+        silence_warnings do 
           require 'rjb'
           Rjb::load(nil, ['-Xms256M', '-Xmx1024M'])
           Rjb::add_jar('/ruby/treat/bin/treat/treat.jar')
@@ -23,7 +23,7 @@ module Treat
           properties.set_property('ner.model.3class', '/ruby/treat/bin/stanford/classifiers/all.3class.distsim.crf.ser.gz')
           properties.set_property('ner.model.7class', '/ruby/treat/bin/stanford/classifiers/muc.7class.distsim.crf.ser.gz')
           properties.set_property('ner.model.MISCclass', '/ruby/treat/bin/stanford/classifiers/conll.4class.distsim.crf.ser.gz')
-          properties.set_property('parser.model', '/ruby/treat/bin/stanford_parser/grammar/englishPCFG.ser.gz')
+          properties.set_property('parser.model', '/ruby/treat/bin/stanford-parser/grammar/englishPCFG.ser.gz')
           silence_stream(STDOUT) do
             pipeline = StanfordCoreNLP.new(properties)
           end

data/lib/treat/extractors/time/chronic.rb

@@ -2,9 +2,9 @@ module Treat
   module Extractors
     module Time
       class Chronic
-        silently { require 'chronic' }
+        silence_warnings { require 'chronic' }
         def self.time(entity, options = {})
-          silently { ::Chronic.parse(entity.to_s, {:guess => true}) }
+          silence_warnings { ::Chronic.parse(entity.to_s, {:guess => true}) }
         end
       end
     end

data/lib/treat/extractors/time/nickel.rb

@@ -15,9 +15,9 @@ module Treat
 =end
       module Nickel
         require 'date'
-        silently { require 'nickel' }
+        silence_warnings { require 'nickel' }
         def self.time(entity, options = {})
-          n = silently { ::Nickel.parse(entity.to_s) }
+          n = silence_warnings { ::Nickel.parse(entity.to_s) }
           occ = n.occurrences[0]
           # Find the words..
           rec = occ.type.to_s.gsub('single', 'once').intern

data/lib/treat/extractors/topic_words/lda.rb

@@ -11,12 +11,12 @@ module Treat
       # Machine Learning Research. 3 (Mar. 2003), 993-1022.
       class LDA
         # Require the lda-ruby gem.
-        silently { require 'lda-ruby' }
+        silence_warnings { require 'lda-ruby' }
         # Monkey patch the TextCorpus class to call it without
         # having to create any files.
         Lda::TextCorpus.class_eval do
           # Ruby, Y U NO SHUT UP!
-          silently { undef :initialize }
+          silence_warnings { undef :initialize }
           # Redefine initialize to take in an array of texts.
           def initialize(texts)
             super(nil)

data/lib/treat/extractors.rb

@@ -1,11 +1,11 @@
 module Treat
   # Extractors extract specific information out of texts.
   module Extractors
-    # Extracts a DateTime object containing a timestamp
-    # from string representation of date/time.
+    # Extracts the time of an object and annotates it
+    # with specific information regarding time.
     module Time
       extend Group
-      self.type = :computer
+      self.type = :annotator
       self.targets = [:word, :constituent, :symbol]
     end
     # Extract the topic from a text.
@@ -20,22 +20,25 @@ module Treat
       self.type = :annotator
       self.targets = [:collection, :document, :text, :zone, :sentence]
     end
-    module Statistics
-      extend Group
-      self.type = :computer
-      self.targets = [:entity]
-      self.default = :none
-    end
+    # Extract named entities from texts.
     module NamedEntity
       extend Group
       self.type = :computer
       self.targets = [:entity]
     end
+    # Extract the key sentences from a text.
     module KeySentences
       extend Group
       self.type = :computer
       self.targets = [:collection, :document, :text, :zone, :sentence]
     end
+    # This module should be moved out of here ASAP.
+    module Statistics
+      extend Group
+      self.type = :computer
+      self.targets = [:entity]
+      self.default = :none
+    end
     extend Treat::Category
   end
 end

data/lib/treat/feature.rb

@@ -1,4 +1,9 @@
 module Treat
+  # This class represents a probabilistic feature;
+  # it is currently not used, because its
+  # behaviour is non-deterministic. Perhaps at 
+  # some point this will be of value for specific
+  # algorithms and so I'm keeping it here.
   class Feature
     # Undefine all methods, except those that 
     # create any problems (e.g. with serializing).
@@ -26,7 +31,7 @@ module Treat
       end
     end
     # Normalize the probabilities, so that
-    # the sum of all probabilities is one,
+    # the sum of all probabilities is 1,
     # except if the sum of all probabilities
     # is already below one (in which case we
     # assume that the feature is intentionally

data/lib/treat/formatters/cleaners/html.rb

@@ -2,7 +2,7 @@ module Treat
   module Formatters
     module Cleaners
       class HTML
-        silently { require 'hpricot' }
+        silence_warnings { require 'hpricot' }
         def self.clean(document, options = {})
           document.each_text do |text|
             text.set :html_value, text.value

data/lib/treat/formatters.rb

@@ -10,6 +10,13 @@ module Treat
       self.targets = [:collection, :document]
       self.default = :autoselect
     end
+    # Serializers transform entities into a storable format.
+    module Serializers
+      extend Group
+      self.type = :computer
+      self.targets = [:entity]
+      self.default = :yaml
+    end
     # Unserializers recreate entities from a serialized format.
     module Unserializers
       extend Group
@@ -24,14 +31,7 @@ module Treat
       self.targets = [:entity]
       self.default = :tree
     end
-    # Serializers transform entities into a storable format.
-    module Serializers
-      extend Group
-      self.type = :computer
-      self.targets = [:entity]
-      self.default = :yaml
-    end
-    # Serializers transform entities into a storable format.
+    # Cleaners strip a text from its mark up.
     module Cleaners
       extend Group
       self.type = :annotator