treat 1.2.0 → 2.0.0rc1

data/lib/treat/workers/extractors/name_tag/stanford.rb

@@ -1,9 +1,15 @@
-# Detects the named entity tag in sentences by using
-# the stanford-core-nlp gem, which interfaces with
-# the Stanford Deterministic Coreference Resolver.
+# Named entity tag extraction using the Stanford NLP
+# Deterministic Coreference Resolver, which implements a
+# multi-pass sieve coreference resolution (or anaphora 
+# resolution) system.
+#
+# Original paper: Heeyoung Lee, Yves Peirsman, Angel 
+# Chang, Nathanael Chambers, Mihai Surdeanu, Dan Jurafsky. 
+# Stanford's Multi-Pass Sieve Coreference Resolution 
+# System at the CoNLL-2011 Shared Task. In Proceedings 
+# of the CoNLL-2011 Shared Task, 2011.
 class Treat::Workers::Extractors::NameTag::Stanford
 
-  require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load
   
   @@classifiers = {}

data/lib/treat/workers/extractors/similarity/levenshtein.rb

@@ -0,0 +1,36 @@
+class Treat::Workers::Extractors::Similarity
+  # Default options.
+  DefaultOptions = {
+    with: '',
+    ins_cost: 1,
+    del_cost: 1,
+    sub_cost: 1
+  }
+  # Return the levensthein distance between
+  # two strings taking into account the costs
+  # of insertion, deletion, and substitution.
+  # Used by did_you_mean? to detect typos.
+  def self.similarity(entity, options)
+    first, other = entity.to_s, options[:with].to_s
+    options = DefaultOptions.merge(options)
+    other, ins, del, sub, = options[:with],
+    options[:inst_cost], options[:del_cost],
+    options[:sub_cost]
+    fill, dm = [0] * (first.length - 1).abs,
+    [(0..first.length).collect { |i| i * ins}]
+    for i in 1..other.length
+      dm[i] = [i * del, fill.flatten]
+    end
+    for i in 1..other.length
+      for j in 1..first.length
+        dm[i][j] = [
+          dm[i-1][j-1] + (first[i-1] == 
+          other[i-1] ? 0 : sub), dm[i][j-1] + 
+          ins, dm[i-1][j] + del
+        ].min
+      end
+    end
+    dm[other.length][first.length]
+  end
+
+end

data/lib/treat/workers/extractors/similarity/tf_idf.rb

@@ -0,0 +1,27 @@
+# Calculates the TF*IDF score of words.
+class Treat::Workers::Extractors::Similarity::TfIdf
+
+  require 'tf-idf-similarity'
+
+  @collections = {}
+  
+  def self.tf_idf(collection, options={})
+    coll = TfIdfSimilarity::Collection.new
+    collection.each_document do |doc|
+      tdoc = TfIdfSimilarity::Document.new(doc.to_s)
+      term_counts = Hash.new(0)
+      doc.each_word do |word| 
+        val = word.value.downcase
+        term_counts[val] ||= 0.0
+        term_counts[val] += 1.0
+      end
+      size = term_counts.values.reduce(:+)
+      tdoc.instance_eval do
+        @term_counts, @size = term_counts, size
+      end
+      coll << tdoc
+    end
+    puts coll.similarity_matrix.inspect
+  end
+
+end

data/lib/treat/workers/extractors/tf_idf/native.rb

@@ -1,5 +1,5 @@
 # Calculates the TF*IDF score of words.
-module Treat::Workers::Extractors::TfIdf::Native
+class Treat::Workers::Extractors::TfIdf::Native
   DefaultOptions = {
     :tf => :natural,
     :idf => :logarithm,
@@ -24,9 +24,9 @@ module Treat::Workers::Extractors::TfIdf::Native
   @@wc = {} # Number of words in a given document (word count).
   @@cw = {} # Common words to filter out.
   def self.tf_idf(entity, options={})
-    l = Treat.languages.send(entity.language)
-    if l.stop_words
-      @@cw[entity.language] = l.stop_words.list
+    l = Treat.languages[entity.language]
+    if l.respond_to?(:stop_words)
+      @@cw[entity.language] = l.stop_words
       return 0 if @@cw[entity.language].include?(entity.value)
     end
     return 0 if entity.value.length <= 2

data/lib/treat/workers/extractors/time/chronic.rb

@@ -1,7 +1,5 @@
-# A wrapper for the 'chronic' gem, which parses
-# date information.
-#
-# Project website: http://chronic.rubyforge.org/
+# Time/date extraction using a rule-based, pure 
+# Ruby natural language date parser.
 class Treat::Workers::Extractors::Time::Chronic
 
   # Require the 'chronic' gem.

data/lib/treat/workers/extractors/time/nickel.rb

@@ -1,23 +1,5 @@
-# A wrapper for the 'nickel' gem, which parses
-# times and dates and supplies additional information
-# concerning these. The additional information supplied
-# that this class annotates entities with is:
-#
-# - time_recurrence: frequency of recurrence in words*.
-# - time_recurrence_interval: frequency of recurrence in days.
-# - start_time: a DateTime object representing the beginning of
-#   an event.
-# - end_time: a DateTime object representing the end of an event.
-#
-# Examples of values for time_recurrence are:
-#
-# - single: "lunch with megan tomorrow at noon"
-# - daily: "Art exhibit until March 1st"
-# - weekly: "math class every wed from 8-11am"
-# - daymonthly: "open bar at joes the first friday of every month"
-# - datemonthly: "pay credit card bill on the 22nd of each month"
-#
-# Project website: http://naturalinputs.com/
+# Time extraction using a pure Ruby natural language
+# time parser.
 class Treat::Workers::Extractors::Time::Nickel
 
   require 'date'
@@ -25,6 +7,23 @@ class Treat::Workers::Extractors::Time::Nickel
   silence_warnings { require 'nickel' }
 
   # Extract time information from a bit of text.
+  #
+  # In addition to the :time annotation, this class will provided:
+  #
+  # - time_recurrence: frequency of recurrence in words*.
+  # - time_recurrence_interval: frequency of recurrence in days.
+  # - start_time: a DateTime object representing the beginning of
+  #   an event.
+  # - end_time: a DateTime object representing the end of an event.
+  #
+  # Examples of values for time_recurrence are:
+  #
+  # - single: "lunch with megan tomorrow at noon"
+  # - daily: "Art exhibit until March 1st"
+  # - weekly: "math class every wed from 8-11am"
+  # - daymonthly: "open bar at joes the first friday of every month"
+  # - datemonthly: "pay credit card bill on the 22nd of each month"
+  #
   def self.time(entity, options = {})
     
     s = entity.to_s

data/lib/treat/workers/extractors/time/ruby.rb

@@ -1,4 +1,5 @@
-# A wrapper for Ruby's native date/time parsing.
+# Date extraction using Ruby's standard library
+# DateTime.parse() method.
 class Treat::Workers::Extractors::Time::Ruby
 
   # Require Ruby's date module.

data/lib/treat/workers/extractors/topic_words/lda.rb

@@ -1,14 +1,14 @@
-# An adapter for the 'lda-ruby' gem, which clusters
-# documents into topics based on Latent Dirichlet
-# Allocation.
+# Topic word retrieval using a thin wrapper over a 
+# C implementation of Latent Dirichlet Allocation (LDA),
+# a statistical model that posits each document 
+# is a mixture of a small number of topics and that 
+# each word's creation is attributable to one of the 
+# document's topics.
 #
-# Original paper:
-# Blei, David M., Ng, Andrew Y., and Jordan, Michael
-# I. 2003. Latent dirichlet allocation. Journal of
+# Original paper: Blei, David, Ng, Andrew, and Jordan, 
+# Michael. 2003. Latent dirichlet allocation. Journal of
 # Machine Learning Research. 3 (Mar. 2003), 993-1022.
-#
-# Project website: https://github.com/ealdent/lda-ruby
-module Treat::Workers::Extractors::TopicWords::LDA
+class Treat::Workers::Extractors::TopicWords::LDA
 
   # Require the lda-ruby gem.
   silence_warnings { require 'lda-ruby' }
@@ -53,9 +53,9 @@ module Treat::Workers::Extractors::TopicWords::LDA
     # Run the EM algorithm using random 
     # starting points
     
-    silence_stdout do
-      lda.em('random')
-    end
+    Treat.core.verbosity.silence ?
+    silence_stdout { lda.em('random') :
+    lda.em('random')
     
     # Load the vocabulary.
     if options[:vocabulary]

data/lib/treat/workers/extractors/topics/reuters.rb

@@ -1,12 +1,9 @@
 # A Ruby text categorizer that was trained using 
-# the Reuters news story corpus. 
+# the Reuters news story corpus. Works well for
+# news articles, not so well for other sources.
 #
-# Copyright 2005 Mark Watson. All rights reserved.
-# Rewrite for inclusion in Treat by Louis Mullie (2011).
-#
-# Original project website: 
-# http://www.markwatson.com/opensource/
-module Treat::Workers::Extractors::Topics::Reuters
+# Authors: Mark Watson, 2005; Louis Mullie, 2011.
+class Treat::Workers::Extractors::Topics::Reuters
 
   # Require the Nokogiri XML parser.
   require 'nokogiri'
@@ -46,12 +43,11 @@ module Treat::Workers::Extractors::Topics::Reuters
   # Read the topics from the XML files.
   def self.get_topics
     return unless @@industry.size == 0
-    @@industry = read_xml(Treat.paths.models + 
-    'reuters/industry.xml')
-    @@region = read_xml(Treat.paths.models + 
-    'reuters/region.xml')
-    @@topics = read_xml(Treat.paths.models + 
-    'reuters/topics.xml')
+    path = (Treat.libraries.reuters.model_path || 
+    (Treat.paths.models + 'reuters/'))
+    @@industry = read_xml(path + 'industry.xml')
+    @@region = read_xml(path + 'region.xml')
+    @@topics = read_xml(path + 'topics.xml')
   end
   
   # Read an XML file and populate a

data/lib/treat/workers/formatters/readers/autoselect.rb

@@ -28,7 +28,7 @@ class Treat::Workers::Formatters::Readers::Autoselect
     format = default_to if format.to_s == ''
     
     begin
-      Treat::Workers::Formatters::Readers.const_get(cc(format))
+      Treat::Workers::Formatters::Readers.const_get(format.cc)
     rescue Treat::Exception
       format = default_to
     end

data/lib/treat/workers/formatters/readers/image.rb

@@ -7,9 +7,8 @@
 # statistical natural language modeling, and multi-
 # lingual capabilities."
 #
-# Original paper:
-#
-# Breuel, Thomas M. The Ocropus Open Source OCR System.
+# Original paper: Google Ocropus Engine: Breuel, 
+# Thomas M. The Ocropus Open Source OCR System. 
 # DFKI and U. Kaiserslautern, Germany.
 class Treat::Workers::Formatters::Readers::Image
 
@@ -21,24 +20,35 @@ class Treat::Workers::Formatters::Readers::Image
   def self.read(document, options = {})
     
     read = lambda do |doc|
-      create_temp_dir do |tmp|
+      self.create_temp_dir do |tmp|
         `ocropus book2pages #{tmp}/out #{doc.file}`
         `ocropus pages2lines #{tmp}/out`
         `ocropus lines2fsts #{tmp}/out`
         `ocropus buildhtml #{tmp}/out > #{tmp}/output.html`
         doc.set :file,  "#{tmp}/output.html"
+        doc.set :format, :html
         doc = doc.read(:html)
-        doc.set :file, f
-        doc.set :format, 'image'
       end
     end
     
-    options[:silent] ?
-    silence_stdout { read.call(document) } :
-    read.call(document)
+    Treat.core.verbosity.silence ? silence_stdout { 
+    read.call(document) } : read.call(document)
     
     document
     
   end
   
+  # Create a dire that gets deleted after execution of the block.
+  def self.create_temp_dir(&block)
+    if not FileTest.directory?(Treat.paths.tmp)
+      FileUtils.mkdir(Treat.paths.tmp)
+    end
+    dname = Treat.paths.tmp +
+    "#{Random.rand(10000000).to_s}"
+    Dir.mkdir(dname)
+    block.call(dname)
+  ensure
+    FileUtils.rm_rf(dname)
+  end
+  
 end

data/lib/treat/workers/formatters/readers/odt.rb

@@ -24,7 +24,8 @@ class Treat::Workers::Formatters::Readers::ODT
     Zip::ZipFile::CREATE) do |zipfile|
       f = zipfile.read('content.xml')
     end
-    raise "Couldn't unzip dot file " +
+    raise Treat::Exception, 
+    "Couldn't unzip dot file " +
     "#{document.file}!" unless f
     xml_h = ODTXmlHandler.new
     REXML::Document.parse_stream(f, xml_h)

data/lib/treat/workers/formatters/readers/pdf.rb

@@ -1,15 +1,16 @@
 # encoding: utf-8
 # A wrapper for the Poppler pdf2text utility, which
 # extracts the text from a PDF file.
-module Treat::Workers::Formatters::Readers::PDF
+class Treat::Workers::Formatters::Readers::PDF
 
+  require 'fileutils'
+  
   # Read a PDF file using the Poppler pdf2text utility.
   #
   # Options: none.
   def self.read(document, options = {})
     
-    create_temp_file(:txt) do |tmp|
-      
+    self.create_temp_file(:txt) do |tmp|
       `pdftotext #{document.file} #{tmp} `.strip
       f = File.read(tmp)
       f.gsub!("\t\r ", '')
@@ -27,5 +28,21 @@ module Treat::Workers::Formatters::Readers::PDF
     end
     
   end
+  
+  # Create a temporary file which is deleted
+  # after execution of the block.
+  def self.create_temp_file(ext, value = nil, &block)
+    if not FileTest.directory?(Treat.paths.tmp)
+      FileUtils.mkdir(Treat.paths.tmp)
+    end
+    fname = Treat.paths.tmp + 
+    "#{Random.rand(10000000).to_s}.#{ext}"
+    File.open(fname, 'w') do |f|
+      f.write(value) if value
+      block.call(f.path)
+    end
+  ensure
+    File.delete(fname)
+  end
 
 end

data/lib/treat/workers/formatters/readers/xml.rb

@@ -1,6 +1,5 @@
 class Treat::Workers::Formatters::Readers::XML
 
-  require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load
   require 'cgi'
 

data/lib/treat/workers/formatters/serializers/mongo.rb

@@ -1,4 +1,4 @@
-# Stores an entity in a Mongo collection.
+# Serialization of entities to a Mongo database.
 class Treat::Workers::Formatters::Serializers::Mongo
 
   # Reauire the Mongo DB
@@ -16,7 +16,9 @@ class Treat::Workers::Formatters::Serializers::Mongo
     Treat::Entities.const_get(
     options[:stop_at].to_s.capitalize) : nil
 
-    if !Treat.databases.mongo.db && !options[:db]
+    options[:db] ||= Treat.databases.mongo.db
+    
+    if !options[:db]
       raise Treat::Exception,
       'Must supply the database name in config. ' +
       '(Treat.databases.mongo.db = ...) or pass ' +
@@ -25,30 +27,18 @@ class Treat::Workers::Formatters::Serializers::Mongo
 
     @@database ||= Mongo::Connection.
     new(Treat.databases.mongo.host).
-    db(options[:db] || Treat.databases.mongo.db)
+    db(options[:db])
 
-    supertype =  cl(Treat::Entities.const_get(
-    entity.type.to_s.capitalize.intern).superclass).downcase
+    supertype =  Treat::Entities.const_get(
+    entity.type.to_s.capitalize.intern).superclass.mn.downcase
     supertype = entity.type.to_s if supertype == 'entity'
     supertypes = supertype + 's'
     
     coll = @@database.collection(supertypes)
+    entity_token = self.do_serialize(entity, options)
+    coll.update({id: entity.id}, entity_token, {upsert: true})
     
-    if entity.type == :collection
-      docs = @@database.collection('documents')                    # Take a design decision here.
-      coll.update(
-        {id: entity.id}, self.do_serialize(entity, 
-        options.merge({:stop_at => Treat::Entities::Document})), 
-        {upsert: true})
-      entity.each_document do |doc|
-        docs.update(
-        {id: doc.id}, self.do_serialize(doc, options), 
-        {upsert: true})
-      end
-    else
-      entity_token = self.do_serialize(entity, options)
-      coll.update({id: entity.id}, entity_token, {upsert: true})
-    end
+    {db: options[:db], collection: supertypes, id: entity.id}
     
   end
 

data/lib/treat/workers/formatters/serializers/xml.rb

@@ -1,4 +1,4 @@
-# This class converts an entity to a storable XML format.
+# Serialization of entities to XML format.
 class Treat::Workers::Formatters::Serializers::XML
 
   # Reauire the Nokogiri XML parser.
@@ -8,18 +8,20 @@ class Treat::Workers::Formatters::Serializers::XML
   # Options:
   # - (String) :file => a file to write to.
   def self.serialize(entity, options = {})
-    if options[:indent].nil?
-      options = options.merge({:indent => 0})
-    end
-    indent = options[:indent]
-    if options[:indent] == 0
-      enc = entity.to_s.encoding.to_s.downcase
-      string = "<?xml version=\"1.0\" " +
-      "encoding=\"#{enc}\" ?>\n<treat>\n"
-    else
-      string = ''
-    end
-    spaces = ''
+    options[:file] ||= (entity.id.to_s + '.xml')
+    options[:indent] = 0
+    enc = entity.to_s.encoding.to_s.downcase
+    string = "<?xml version=\"1.0\" " +
+    "encoding=\"#{enc}\" ?>\n<treat>\n"
+    val = self.recurse(entity, options)
+    string += "#{val}\n</treat>"
+    File.open(options[:file], 'w') do |f|
+      f.write(string)
+    end; return string
+  end
+  
+  def self.recurse(entity, options)
+    spaces, string = '', ''
     options[:indent].times { spaces << ' ' }
     attributes = " id='#{entity.id}'"
     if !entity.features.nil? && entity.features.size != 0
@@ -55,27 +57,16 @@ class Treat::Workers::Formatters::Serializers::XML
     if entity.has_children?
       options[:indent] += 1
       entity.children.each do |child|
-        string =
-        string +
-        serialize(child, options)
+        string += self.recurse(child, options)
       end
       options[:indent] -= 1
     else
-      string = string + "#{escape(entity.value)}"
+      string += "#{escape(entity.value)}"
     end
     unless entity.is_a?(Treat::Entities::Token)
       string += "#{spaces}"
     end
     string += "</#{tag}>\n"
-    if indent == 0
-      string += "\n</treat>"
-      if options[:file]
-        File.open(options[:file], 'w') do |f|
-          f.write(string)
-        end
-      end
-    end
-    string
   end
 
   def self.escape(input)

data/lib/treat/workers/formatters/serializers/yaml.rb

@@ -1,9 +1,9 @@
-# This class serializes entities in YAML format.
+# Serialization of entities to YAML format.
 class Treat::Workers::Formatters::Serializers::YAML
   
   silence_warnings do
     # Require the Psych YAML serializer.
-    require 'psych'
+    require 'yaml'
   end
   
   # Serialize an entity in YAML format.
@@ -11,13 +11,14 @@ class Treat::Workers::Formatters::Serializers::YAML
   # Options:
   # - (String) :file => a file to write to.
   def self.serialize(entity, options = {})
-    yaml = ::Psych.dump(entity)
+    yaml = ::YAML.dump(entity)
+    options[:file] ||= (entity.id.to_s + '.yml')
     if options[:file]
       File.open(options[:file], 'w') do |f| 
         f.write(yaml)
       end
     end
-    yaml
+    options[:file]
   end
 
 end

data/lib/treat/workers/formatters/unserializers/mongo.rb

@@ -1,4 +1,5 @@
-module Treat::Workers::Formatters::Unserializers::Mongo
+# Unserialization of entities stored in a Mongo database.
+class Treat::Workers::Formatters::Unserializers::Mongo
   
   require 'mongo'
   
@@ -18,8 +19,8 @@ module Treat::Workers::Formatters::Unserializers::Mongo
     new(Treat.databases.mongo.host).
     db(Treat.databases.mongo.db || db)
     
-    supertype =  cl(Treat::Entities.const_get(
-    entity.type.to_s.capitalize.intern).superclass).downcase
+    supertype =  Treat::Entities.const_get(
+    entity.type.to_s.capitalize.intern).superclass.mn.downcase
     supertype = entity.type.to_s if supertype == 'entity'
     supertypes = supertype + 's'
     supertypes = 'documents' if entity.type == :collection
@@ -58,7 +59,6 @@ module Treat::Workers::Formatters::Unserializers::Mongo
     const_get(record['type'].
     capitalize.intern).new(
     record['value'], record['id'])
-    
     features = record['features']
     new_feat = {}
     features.each do |feature, value|

data/lib/treat/workers/formatters/unserializers/xml.rb

@@ -1,6 +1,5 @@
-# Recreates the entity tree corresponding to
-# a serialized XML file.
-module Treat::Workers::Formatters::Unserializers::XML
+# Unserialization of entities stored in XML format.
+class Treat::Workers::Formatters::Unserializers::XML
 
   require 'nokogiri'
   
@@ -106,7 +105,7 @@ module Treat::Workers::Formatters::Unserializers::XML
   end
 
   def self.revive(type, value, id)
-    klass = Treat::Entities.const_get(cc(type))
+    klass = Treat::Entities.const_get(type.cc)
     klass.new(value, id)
   end
 

data/lib/treat/workers/formatters/unserializers/yaml.rb

@@ -1,10 +1,9 @@
-# This class is a wrapper for the Psych YAML
-# parser; it unserializes YAML files.
+# Unserialization of entities stored in YAML format.
 class Treat::Workers::Formatters::Unserializers::YAML
 
   silence_warnings do
     # Require the Psych YAML parser.
-    require 'psych'
+    require 'yaml'
   end
   
   # Require date to revive DateTime.
@@ -14,7 +13,7 @@ class Treat::Workers::Formatters::Unserializers::YAML
   #
   # Options: none.
   def self.unserialize(document, options = {})
-    document << ::Psych.load(
+    document << ::YAML.load(
     File.read(document.file))
     document
   end

data/lib/treat/workers/formatters/visualizers/dot.rb

@@ -1,3 +1,4 @@
+# Visualization of entities in DOT graph format.
 class Treat::Workers::Formatters::Visualizers::DOT
 
   require 'date'

data/lib/treat/workers/formatters/visualizers/standoff.rb

@@ -1,6 +1,5 @@
-# This class allows the visualization of
-# an entity in standoff format; for example:
-# (S (NP John) (VP has (VP come))).
+# Visualization of entities in standoff (tag-bracketed)
+# format, based on the Stanford tag-bracketed format.
 class Treat::Workers::Formatters::Visualizers::Standoff
 
   # Start out with an indent of 0.

data/lib/treat/workers/formatters/visualizers/tree.rb

@@ -1,9 +1,8 @@
-# This class generates an ASCII representation
-# of a tree of entities.
+# Visualization of entities in ASCII tree format.
 class Treat::Workers::Formatters::Visualizers::Tree
 
   # Start out with an indent at 0.
-  DefaultOptions = { :indent => 0 }
+  DefaultOptions = { indent: 0 }
   
   # Obtain a plain text tree representation
   # of the entity.