treat 1.0.6 → 1.1.0

data/lib/treat/proxies.rb

@@ -1,57 +1,106 @@
-# Proxies install builders on core Ruby objects,
-# so that methods called on them may be passed
-# to the entity that can be built from the core
-# class instance.
-module Treat::Proxies
-  
+# Proxies install builders on core Ruby objects;
+# when a method defined by Treat is called on these
+# objects, the Ruby object is cast to a Treat entity
+# and the method is called on the resultant type.
+module Treat::Core::Proxies
+
   # Provides a base functionality for proxies.
   module Proxy
-    
+
     # Build the entity corresponding to the proxied
     # object and send the method call to the entity.
     def method_missing(sym, *args, &block)
-      if sym == :do || Treat::Categories.lookup(sym)
+      if sym == :do || Treat::Workers.lookup(sym)
         to_entity.send(sym, *args)
       else
         super(sym, *args, &block)
       end
     end
-    
+
     # Create an unknown type of entity by default.
     def to_entity(builder = nil)
       Treat::Entities::Unknown(self.to_s)
     end
-    
+
   end
-  
+
   # Install Treat functions on String objects.
   module String
-    
+
     # Include base proxy functionality.
-    include Treat::Proxies::Proxy
-    
+    include Treat::Core::Proxies::Proxy
+
     # Return the entity corresponding to the string.
     def to_entity
-      Treat::Entities::Entity.from_string(self.to_s)
+      Treat::Entities::Entity.from_string(self)
     end
-    
+
   end
-  
+
   # Install Treat functions on Numeric objects.
   module Numeric
-    
+
     # Include base proxy functionality.
-    include Treat::Proxies::Proxy
-    
+    include Treat::Core::Proxies::Proxy
+
     # Return the entity corresponding to the number.
     def to_entity(builder = nil)
       Treat::Entities::Number.from_numeric(self)
     end
+
+  end
   
+  # Include Treat methods on strings.
+  ::String.class_eval do
+    include Treat::Core::Proxies::String
   end
   
-  # Include the proxies in the core classes.
-  ::String.class_eval { include Treat::Proxies::String }
-  ::Numeric.class_eval { include Treat::Proxies::Numeric }
+  # Include Treat methods on numerics.
+  ::Numeric.class_eval do
+    include Treat::Core::Proxies::Numeric
+  end
+  
+  # This is kind of ugly; need to find a
+  # better solution eventually (?)
+  Treat::Entities::Entity.class_eval do
+    
+    # Rename the true language detection
+    # method to :language_proxied, and
+    # only call it if language detection
+    # is turned on in the configuration.
+    alias :language_proxied :language
+    
+    # Proxy the #language method, defined on
+    # all textual entities, in order to catch
+    # the method call if language detection is
+    # turned off and return the default language
+    # in that case.
+    def language(extractor = nil, options = {})
   
-end
+      return Treat.core.language.default if
+      !Treat.core.language.detect
+
+      if is_a?(Treat::Entities::Symbol) ||
+        is_a?(Treat::Entities::Number)
+        return Treat.core.language.default
+      end
+
+      dlvl = Treat.core.language.detect_at
+      dklass = Treat::Entities.const_get(cc(dlvl))
+      
+      if self.class.compare_with(
+        dklass) < 1 && has_parent?
+        anc = ancestor_with_type(dlvl)
+        return anc.language if anc
+      end
+
+      extractor ||= Treat.workers.
+      extractors.language.default
+
+      language_proxied(extractor, options)
+
+    end
+
+  end
+
+end

data/lib/treat/version.rb

@@ -0,0 +1,3 @@
+module Treat
+  VERSION = "1.1.0"
+end

data/lib/treat/{extractors → workers/extractors}/keywords/tf_idf.rb

@@ -1,7 +1,7 @@
 # This retrieves a supplied number of keywords
 # by selecting the N words with the highest TF*IDF
 # for each document.
-class Treat::Extractors::Keywords::TfIdf
+class Treat::Workers::Extractors::Keywords::TfIdf
   
   # Default options - retrieve 5 keywords.
   DefaultOptions = { :number => 5 }

data/lib/treat/{extractors → workers/extractors}/language/what_language.rb

@@ -1,4 +1,4 @@
-module Treat::Extractors::Language
+module Treat::Workers::Extractors::Language
   
   # Adaptor for the 'whatlanguage' gem, which
   # performs probabilistic language detection.
@@ -15,7 +15,7 @@ module Treat::Extractors::Language
     
     # By default, bias towards common languages.
     DefaultOptions = {
-      :bias => [:eng, :fre, :chi, :ger, :ara, :spa]
+      :bias_toward => [:english, :french, :chinese, :german, :arabic, :spanish]
     }
     
     # Keep only once instance of the gem class.
@@ -32,21 +32,28 @@ module Treat::Extractors::Language
     # toward when more than one language is detected
     # with equal probability.
     def self.language(entity, options = {})
+      
       options = DefaultOptions.merge(options)
+      
       @@detector ||= ::WhatLanguage.new(:possibilities)
       possibilities = @@detector.process_text(entity.to_s)
       lang = {}
+
       possibilities.each do |k,v|
-        lang[Treat::Languages.code(k)] = v
+        lang[k.intern] = v
       end
+      
       max = lang.values.max
       ordered = lang.select { |i,j| j == max }.keys
+      
       ordered.each do |l|
-        if options[:bias].include?(l)
+        if options[:bias_toward].include?(l)
           return l
         end
       end
+      
       return ordered.first
+      
     end
     
   end

data/lib/treat/{extractors → workers/extractors}/name_tag/stanford.rb

@@ -1,7 +1,7 @@
 # Detects the named entity tag in sentences by using
 # the stanford-core-nlp gem, which interfaces with
 # the Stanford Deterministic Coreference Resolver.
-class Treat::Extractors::NameTag::Stanford
+class Treat::Workers::Extractors::NameTag::Stanford
 
   require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load
@@ -13,9 +13,8 @@ class Treat::Extractors::NameTag::Stanford
     pp = nil
 
     lang = entity.language
-    
-    language = Treat::Languages.describe(lang)
-    Treat::Loaders::Stanford.load(language)
+  
+    Treat::Loaders::Stanford.load(lang)
     
     isolated_token = entity.is_a?(Treat::Entities::Token)
     tokens = isolated_token ? [entity] : entity.tokens

data/lib/treat/{extractors → workers/extractors}/tf_idf/native.rb

@@ -1,5 +1,5 @@
 # Calculates the TF*IDF score of words.
-module Treat::Extractors::TfIdf::Native
+module Treat::Workers::Extractors::TfIdf::Native
   DefaultOptions = {
     :tf => :natural,
     :idf => :logarithm,
@@ -24,10 +24,9 @@ module Treat::Extractors::TfIdf::Native
   @@wc = {} # Number of words in a given document (word count).
   @@cw = {} # Common words to filter out.
   def self.tf_idf(entity, options={})
-    l = Treat::Languages.get(entity.language)
-    if l.const_defined?(:CommonWords)
-      @@cw[entity.language] = 
-      l.const_get(:CommonWords)
+    l = Treat.languages.send(entity.language)
+    if l.stop_words
+      @@cw[entity.language] = l.stop_words.list
       return 0 if @@cw[entity.language].include?(entity.value)
     end
     return 0 if entity.value.length <= 2

data/lib/treat/{extractors → workers/extractors}/time/chronic.rb

@@ -2,7 +2,7 @@
 # date information.
 #
 # Project website: http://chronic.rubyforge.org/
-class Treat::Extractors::Time::Chronic
+class Treat::Workers::Extractors::Time::Chronic
 
   # Require the 'chronic' gem.
   silence_warnings { require 'chronic' }

data/lib/treat/{extractors → workers/extractors}/time/nickel.rb

@@ -18,7 +18,7 @@
 # - datemonthly: "pay credit card bill on the 22nd of each month"
 #
 # Project website: http://naturalinputs.com/
-class Treat::Extractors::Time::Nickel
+class Treat::Workers::Extractors::Time::Nickel
 
   require 'date'
 

data/lib/treat/{extractors → workers/extractors}/time/ruby.rb

@@ -1,5 +1,5 @@
 # A wrapper for Ruby's native date/time parsing.
-class Treat::Extractors::Time::Ruby
+class Treat::Workers::Extractors::Time::Ruby
 
   # Require Ruby's date module.
   require 'date'

data/lib/treat/{extractors → workers/extractors}/topic_words/lda.rb

@@ -8,7 +8,7 @@
 # Machine Learning Research. 3 (Mar. 2003), 993-1022.
 #
 # Project website: https://github.com/ealdent/lda-ruby
-module Treat::Extractors::TopicWords::LDA
+module Treat::Workers::Extractors::TopicWords::LDA
 
   # Require the lda-ruby gem.
   silence_warnings { require 'lda-ruby' }

data/lib/treat/{extractors → workers/extractors}/topics/reuters.rb

@@ -6,7 +6,7 @@
 #
 # Original project website: 
 # http://www.markwatson.com/opensource/
-module Treat::Extractors::Topics::Reuters
+module Treat::Workers::Extractors::Topics::Reuters
 
   # Require the Nokogiri XML parser.
   require 'nokogiri'
@@ -46,11 +46,11 @@ module Treat::Extractors::Topics::Reuters
   # Read the topics from the XML files.
   def self.get_topics
     return unless @@industry.size == 0
-    @@industry = read_xml(Treat.models + 
+    @@industry = read_xml(Treat.paths.models + 
     'reuters/industry.xml')
-    @@region = read_xml(Treat.models + 
+    @@region = read_xml(Treat.paths.models + 
     'reuters/region.xml')
-    @@topics = read_xml(Treat.models + 
+    @@topics = read_xml(Treat.paths.models + 
     'reuters/topics.xml')
   end
   

data/lib/treat/{formatters → workers/formatters}/readers/abw.rb

@@ -8,7 +8,7 @@
 # Todo: reimplement with Nokogiri and use
 # XML node information to better translate
 # the format of the text.
-class Treat::Formatters::Readers::ABW
+class Treat::Workers::Formatters::Readers::ABW
 
   silence_warnings do
     require 'rexml/document'
@@ -25,7 +25,7 @@ class Treat::Formatters::Readers::ABW
     IO.read(document.file), xml_h)
 
     document.value = xml_h.plain_text
-    document.set :format, :abw_word
+    document.set :format, 'abw'
     document
 
   end

data/lib/treat/{formatters → workers/formatters}/readers/autoselect.rb

@@ -1,9 +1,9 @@
-class Treat::Formatters::Readers::Autoselect
+class Treat::Workers::Formatters::Readers::Autoselect
   
   ExtensionRegexp = /^.*?\.([a-zA-Z0-9]{2,5})$/
   ImageExtensions = ['gif', 'jpg', 'jpeg', 'png']
   DefaultOptions = {
-    :default_to => :txt
+    :default_to => 'txt'
   }
   
   # Choose a reader to use.
@@ -16,6 +16,7 @@ class Treat::Formatters::Readers::Autoselect
   end
   
   def self.detect_format(filename, default_to = nil)
+    
     default_to ||= DefaultOptions[:default_to]
     ext = filename.scan(ExtensionRegexp)
     ext = (ext.is_a?(Array) && ext[0] && ext[0][0]) ? ext[0][0] : ''
@@ -25,7 +26,13 @@ class Treat::Formatters::Readers::Autoselect
     format = 'yaml' if format == 'yml'
 
     format = default_to if format.to_s == ''
-
+    
+    begin
+      Treat::Workers::Formatters::Readers.const_get(cc(format))
+    rescue Treat::Exception
+      format = default_to
+    end
+    
     format.intern
 
   end

data/lib/treat/{formatters → workers/formatters}/readers/doc.rb

@@ -1,5 +1,5 @@
 # A wrapper for the 'antiword' command-line utility.
-class Treat::Formatters::Readers::DOC
+class Treat::Workers::Formatters::Readers::DOC
   
   # Extract the readable text from a DOC file
   # using the antiword command-line utility.
@@ -13,7 +13,7 @@ class Treat::Formatters::Readers::DOC
     f.gsub!('#keep#', "\n\n")
     
     document.value = f
-    document.set :format, :doc
+    document.set :format, 'doc'
     document
     
   end

data/lib/treat/{formatters → workers/formatters}/readers/html.rb

@@ -4,8 +4,8 @@
 #
 # Project homepage:
 # https://github.com/iterationlabs/ruby-readability
-class Treat::Formatters::Readers::HTML
-
+class Treat::Workers::Formatters::Readers::HTML
+  
   silence_warnings { require 'ruby-readability' }
 
   # By default, don't backup the original HTML
@@ -45,9 +45,9 @@ class Treat::Formatters::Readers::HTML
       html.gsub!(/<!--[^>]*-->/m, '')
       d = Readability::Document.new(html, options)
       document.value = "<h1>#{d.title}</h1>\n" + d.content
-      document.set :format, :html
+      document.set :format, 'html'
     end
-    
+
     document
 
   end

data/lib/treat/{formatters → workers/formatters}/readers/image.rb

@@ -11,7 +11,7 @@
 #
 # Breuel, Thomas M. The Ocropus Open Source OCR System.
 # DFKI and U. Kaiserslautern, Germany.
-class Treat::Formatters::Readers::Image
+class Treat::Workers::Formatters::Readers::Image
 
   #  Read a file using the Google Ocropus reader.
   #
@@ -29,7 +29,7 @@ class Treat::Formatters::Readers::Image
         doc.set :file,  "#{tmp}/output.html"
         doc = doc.read(:html)
         doc.set :file, f
-        doc.set :format, :image
+        doc.set :format, 'image'
       end
     end
     

data/lib/treat/{formatters → workers/formatters}/readers/odt.rb

@@ -10,7 +10,7 @@
 # Todo: reimplement with Nokogiri and use
 # XML node information to better translate
 # the format of the text.
-class Treat::Formatters::Readers::ODT
+class Treat::Workers::Formatters::Readers::ODT
 
   # Require the 'zip' gem to unarchive the ODT files
   silence_warnings { require 'zip' }
@@ -30,7 +30,7 @@ class Treat::Formatters::Readers::ODT
     REXML::Document.parse_stream(f, xml_h)
 
     document.value = xml_h.plain_text
-    document.set :format, :odt_office
+    document.set :format, 'odt'
     document
 
   end

data/lib/treat/{formatters → workers/formatters}/readers/pdf.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 # A wrapper for the Poppler pdf2text utility, which
 # extracts the text from a PDF file.
-module Treat::Formatters::Readers::PDF
+module Treat::Workers::Formatters::Readers::PDF
 
   # Read a PDF file using the Poppler pdf2text utility.
   #
@@ -21,7 +21,7 @@ module Treat::Formatters::Readers::PDF
       f.gsub!('#keep#', "\n\n")
       
       document.value = f
-      document.set :format, :pdf
+      document.set :format, 'pdf'
       document
       
     end

data/lib/treat/{formatters → workers/formatters}/readers/txt.rb

@@ -1,5 +1,5 @@
 # This class simply reads a plain text file.
-class Treat::Formatters::Readers::TXT
+class Treat::Workers::Formatters::Readers::TXT
 
   # Build an entity from a string 
   # in plain text format.
@@ -7,7 +7,7 @@ class Treat::Formatters::Readers::TXT
   # Options: none.
   def self.read(document, options = {})
     document.value = File.read(document.file)
-    document.set :format, :txt
+    document.set :format, 'txt'
     document
   end
   

data/lib/treat/{formatters → workers/formatters}/readers/xml.rb

@@ -1,4 +1,4 @@
-class Treat::Formatters::Readers::XML
+class Treat::Workers::Formatters::Readers::XML
 
   require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load
@@ -70,7 +70,7 @@ class Treat::Formatters::Readers::XML
 
     end
     
-    document.set :format, :xml
+    document.set :format, 'xml'
     document
     
   end

data/lib/treat/workers/formatters/serializers/mongo.rb

@@ -0,0 +1,60 @@
+# Stores an entity in a Mongo collection.
+class Treat::Workers::Formatters::Serializers::Mongo
+
+  # Reauire the Mongo DB
+  require 'mongo'
+
+  DefaultOptions = {
+    :recursive => true,
+    :stop_at => :token
+  }
+  
+  def self.serialize(entity, options = {})
+    
+    options = DefaultOptions.merge(options)
+    stop_at = options[:stop_at] ? 
+    Treat::Entities.const_get(
+    options[:stop_at].to_s.capitalize) : 
+    Treat::Entities::Token
+    
+    if !Treat.databases.mongo.db && !options[:db]
+      raise Treat::Exception,
+      'Must supply the database name in config. ' +
+      '(Treat.databases.mongo.db = ...) or pass ' +
+      'it as a parameter to #serialize.'
+    end
+    
+    @@database ||= Mongo::Connection.
+    new(Treat.databases.mongo.host).
+    db(Treat.databases.mongo.db || options[:db])
+    
+    type = cl(entity.class.superclass).downcase
+    type = entity.type.to_s if type == 'entity'
+    types = type + 's'
+
+    coll = @@database.collection(types)
+    
+    entity_token = {
+      :id => entity.id,
+      :value => entity.value,
+      :string => entity.to_s,
+      :type => entity.type,
+      :children => entity.children.map { |c| [c.id, c.type] },
+      :parent => (entity.has_parent? ? entity.parent.id : nil),
+      :features => entity.features
+    }
+    
+    coll.insert(entity_token)
+
+    if options[:recursive] && entity.has_children?
+      entity.each do |child|
+        next if child.class.compare_with(stop_at) < 0
+        self.serialize(child, options)
+      end
+    end
+     
+  end
+
+end
+
+

data/lib/treat/{formatters → workers/formatters}/serializers/xml.rb

@@ -1,5 +1,5 @@
 # This class converts an entity to a storable XML format.
-class Treat::Formatters::Serializers::XML
+class Treat::Workers::Formatters::Serializers::XML
 
   # Reauire the Nokogiri XML parser.
   require 'nokogiri'
@@ -74,7 +74,6 @@ class Treat::Formatters::Serializers::XML
           f.write(string)
         end
       end
-      # puts string
     end
     string
   end

data/lib/treat/{formatters → workers/formatters}/serializers/yaml.rb

@@ -1,5 +1,5 @@
 # This class serializes entities in YAML format.
-class Treat::Formatters::Serializers::YAML
+class Treat::Workers::Formatters::Serializers::YAML
   
   silence_warnings do
     # Require the Psych YAML serializer.

data/lib/treat/{formatters → workers/formatters}/unserializers/autoselect.rb

@@ -1,4 +1,4 @@
-class Treat::Formatters::Unserializers::Autoselect
+class Treat::Workers::Formatters::Unserializers::Autoselect
 
   def self.unserialize(document, options = {})
     file = document.file
@@ -6,6 +6,8 @@ class Treat::Formatters::Unserializers::Autoselect
       document.unserialize(:yaml, options)
     elsif file.index('xml')
       document.unserialize(:xml, options)
+    elsif file.index('mongo')
+      document.unserialize(:mongo, options)
     else
       raise Treat::Exception,
       "Unreadable serialized format for file #{file}."

data/lib/treat/workers/formatters/unserializers/mongo.rb

@@ -0,0 +1,80 @@
+module Treat::Workers::Formatters::Unserializers::Mongo
+
+  DefaultOptions = { 
+    :recursive => true,
+    :stop_at => nil
+  }
+  
+  require 'mongo'
+  
+  def self.unserialize(entity, options={})
+
+    options = DefaultOptions.merge(options)
+    options[:stop_at] = options[:stop_at] ? 
+    Treat::Entities.const_get(
+    options[:stop_at].to_s.capitalize) : 
+    Treat::Entities::Token
+    
+    if !Treat.databases.mongo.db && !options[:db]
+      raise Treat::Exception,
+      'Must supply the database name in config. ' +
+      '(Treat.databases.mongo.db = ...) or pass ' +
+      'it as a parameter to #unserialize.'
+    end
+    
+    @@database ||= Mongo::Connection.
+    new(Treat.databases.mongo.host).
+    db(Treat.databases.mongo.db || options[:db])
+
+    self.do_unserialize(entity, options)
+    
+  end
+
+  def self.do_unserialize(entity, options)
+    
+    supertype =  cl(Treat::Entities.const_get(
+    entity.type.to_s.capitalize.intern).superclass).downcase
+    supertype = entity.type.to_s if supertype == 'entity'
+    supertypes = supertype + 's'
+    
+    coll = @@database.collection(supertypes)
+    record = coll.find_one(:id => entity.id)
+
+    unless record
+      raise Treat::Exception,
+      "Couldn't find record ID #{entity.id}."
+    end
+    
+    # Convert feature keys to symbols.
+    features = record['features']
+    new_feat = {}
+    features.each do |feature, value|
+      new_feat[feature.intern] = value
+    end
+    entity.features = new_feat
+    
+    # Set the entity's value.
+    entity.value = record['value']
+    
+    if entity.class.compare_with(
+      options[:stop_at]) == 0
+      entity.value = record['string']
+    end
+    
+    return entity unless options[:recursive]
+
+    record['children'].each do |c|
+      cid, ctype = *c
+      cklass = Treat::Entities.const_get(
+      ctype.capitalize.intern)
+      next if cklass.compare_with(
+      options[:stop_at]) < 0
+      entity << self.do_unserialize(
+      cklass.new('', cid), options)
+    end
+
+    entity
+    
+  end
+  
+end

data/lib/treat/{formatters → workers/formatters}/unserializers/xml.rb

@@ -1,6 +1,6 @@
 # Recreates the entity tree corresponding to
 # a serialized XML file.
-module Treat::Formatters::Unserializers::XML
+module Treat::Workers::Formatters::Unserializers::XML
 
   require 'nokogiri'
   
@@ -78,7 +78,7 @@ module Treat::Formatters::Unserializers::XML
       current_value = ''
       type = xml_reader.name.intern
 
-      if Treat::Entities.list.include?(type)
+      if Treat.core.entities.list.include?(type)
         if !current_element
           current_element = self.revive(type, current_value, id)
         else