treat 0.2.5 → 1.0.0

data/lib/treat/extractors/time/chronic.rb

@@ -0,0 +1,55 @@
+# A wrapper for the 'chronic' gem, which parses
+# date information.
+#
+# Project website: http://chronic.rubyforge.org/
+class Treat::Extractors::Time::Chronic
+
+  # Require the 'chronic' gem.
+  silence_warnings { require 'chronic' }
+
+  # Require the Ruby DateTime module
+  require 'date'
+
+  # Return the date information contained within 
+  # the entity by parsing it with the 'chronic' gem.
+  #
+  # Options: none.
+  def self.time(entity, options = {})
+
+    s = entity.to_s
+    return if s =~ /^[0-9]+$/
+    time = nil
+    
+    silence_warnings do
+      time = ::Chronic.parse(s, {:guess => true})
+    end
+    
+    if entity.has_parent? && remove_time_from_ancestors(entity, time)
+      nil
+    else
+      time
+    end
+    
+  end
+  
+  # Keeps the lowest-level time annotations that do
+  # not conflict with a higher time annotation.
+  # Returns true if the entity conflicts with a
+  # higher-level time annotation.
+  def self.remove_time_from_ancestors(entity, time)
+    
+    entity.ancestors_with_type(:phrase).each do |a|
+      
+      next if !a.has?(:time)
+      unless a.get(:time) == time
+        return true
+      end
+      a.unset(:time)
+      
+    end
+    
+    false
+    
+  end
+
+end

data/lib/treat/extractors/time/nickel.rb

@@ -1,71 +1,95 @@
-module Treat
-  module Extractors
-    module Time
-      # A wrapper for the 'nickel' gem, which parses
-      # times and dates and supplies additional information
-      # concerning these. The additional information supplied
-      # that this class annotates entities with is:
-      #
-      # - time_recurrence: frequency of recurrence in words*.
-      # - time_recurrence_interval: frequency of recurrence in days.
-      # - start_time: a DateTime object representing the beginning of
-      #   an event.
-      # - end_time: a DateTime object representing the end of an event.
-      #
-      # Examples of values for time_recurrence are:
-      #
-      # - single: "lunch with megan tomorrow at noon"
-      # - daily: "Art exhibit until March 1st"
-      # - weekly: "math class every wed from 8-11am"
-      # - daymonthly: "open bar at joes the first friday of every month"
-      # - datemonthly: "pay credit card bill on the 22nd of each month"
-      #
-      # Project website: http://naturalinputs.com/
-      class Nickel
-        require 'date'
-        silence_warnings { require 'nickel' }
-        # Extract time information from a bit of text.
-        def self.time(entity, options = {})
-          return nil if entity.to_s.strip == ''
-          n = nil
-          silence_warnings { n = ::Nickel.parse(entity.to_s.strip) }
-          occ = n.occurrences[0]
-          return nil unless occ
+# A wrapper for the 'nickel' gem, which parses
+# times and dates and supplies additional information
+# concerning these. The additional information supplied
+# that this class annotates entities with is:
+#
+# - time_recurrence: frequency of recurrence in words*.
+# - time_recurrence_interval: frequency of recurrence in days.
+# - start_time: a DateTime object representing the beginning of
+#   an event.
+# - end_time: a DateTime object representing the end of an event.
+#
+# Examples of values for time_recurrence are:
+#
+# - single: "lunch with megan tomorrow at noon"
+# - daily: "Art exhibit until March 1st"
+# - weekly: "math class every wed from 8-11am"
+# - daymonthly: "open bar at joes the first friday of every month"
+# - datemonthly: "pay credit card bill on the 22nd of each month"
+#
+# Project website: http://naturalinputs.com/
+class Treat::Extractors::Time::Nickel
 
-          rec = occ.type.to_s.gsub('single', 'once').intern
-          time_recurrence = rec
-          interval = occ.interval ? occ.interval : :none
-          time_recurrence_interval = interval
+  require 'date'
 
-          s = [occ.start_date, occ.start_time]
-          ds = [s[0].year, s[0].month, s[0].day] if s[0]
-          ts = [s[1].hour, s[1].minute, s[1].second] if s[1]
+  silence_warnings { require 'nickel' }
 
-          e = [occ.end_date, occ.end_time]
-          de = [e[0].year, e[0].month, e[0].day] if e[0]
-          te = [e[1].hour, e[1].minute, e[1].second] if e[1]
+  # Extract time information from a bit of text.
+  def self.time(entity, options = {})
+    
+    s = entity.to_s
+    return if s =~ /^[0-9]+$/
+    
+    n = nil
+    
+    begin
+      silence_warnings { n = ::Nickel.parse(s.to_s.strip) }
+    rescue
+      return
+    end
+    
+    occ = n.occurrences[0]
+    
+    return unless occ
+
+    rec = occ.type.to_s.gsub('single', 'once').intern
+    time_recurrence = rec
+    interval = occ.interval ?
+    occ.interval : :none
+    time_recurrence_interval = interval
+
+
+    s = [occ.start_date, occ.start_time]
+    ds = [s[0].year, s[0].month, s[0].day] if s[0]
+    ts = [s[1].hour, s[1].minute, s[1].second] if s[1]
 
-          start_time = ::DateTime.civil(*ds) if ds && !ts
-          start_time = ::DateTime.civil(*ds, *ts) if ds && ts
-          end_time = ::DateTime.civil(*de) if de && !te
-          end_time = ::DateTime.civil(*de, *te) if de && te
+    e = [occ.end_date, occ.end_time]
+    de = [e[0].year, e[0].month, e[0].day] if e[0]
+    te = [e[1].hour, e[1].minute, e[1].second] if e[1]
 
-          time = Treat::Features::Time.new(             # Fix - time message.
-          start_time, end_time, time_recurrence,
-          time_recurrence_interval
-          )
+    start_time = ::DateTime.civil(*ds) if ds && !ts
+    start_time = ::DateTime.civil(*ds, *ts) if ds && ts
+    end_time = ::DateTime.civil(*de) if de && !te
+    end_time = ::DateTime.civil(*de, *te) if de && te
+    
+    return unless start_time
 
-          # Keeps the lowest-level time annotations
-          # that do not conflict with the highest-level
-          # time annotation.
-          entity.ancestors_with_type(:phrase).each do |a|
-            unless a.id == entity.id || a.children[0].size == 0
-              a.unset(:time)
-            end
-          end
-          time
-        end
-      end
+    if entity.has_parent? && 
+      remove_time_from_ancestors(entity, start_time)
+      nil
+    else
+      entity.set :time_recurrence,
+      time_recurrence
+      entity.set :time_recurrence_interval,
+      time_recurrence_interval
+      entity.set :end_time, end_time if end_time
+      start_time
     end
+    
   end
+
+  # Keeps the lowest-level time annotations that do
+  # not conflict with a higher time annotation.
+  # Returns true if the entity conflicts with a
+  # higher-level time annotation.
+  def self.remove_time_from_ancestors(entity, time)
+    entity.ancestors_with_type(:phrase).each do |a|
+      next if !a.has?(:time)
+      return false unless a.get(:time).to_s == time.to_s
+      a.unset(:time, :time_recurrence, 
+      :time_recurrence_interval, :end_time)
+    end
+    true
+  end
+
 end

data/lib/treat/extractors/time/ruby.rb

@@ -0,0 +1,53 @@
+# A wrapper for Ruby's native date/time parsing.
+class Treat::Extractors::Time::Ruby
+
+  # Require Ruby's date module.
+  require 'date'
+
+  # Return a DateTime object representing the date/time
+  # contained within the entity, using Ruby's native
+  # date/time parser. This extractor is suitable for the
+  # detection of well-structured dates and times, such as
+  # 2011/02/03 5:00.
+  #
+  # Options: none.
+  def self.time(entity, options = {})
+    s = entity.to_s
+    return if s =~ /^[0-9]+$/
+    begin
+      time = ::DateTime.parse(s)
+      if  entity.has_parent? && 
+        remove_time_from_ancestors(entity, time)
+        nil
+      else
+        time
+      end
+    rescue
+      nil
+    end
+  end
+
+  
+  # Keeps the lowest-level time annotations that do
+  # not conflict with a higher time annotation.
+  # Returns true if the entity conflicts with a
+  # higher-level time annotation.
+  def self.remove_time_from_ancestors(entity, time)
+    
+    entity.ancestors_with_type(:phrase).each do |a|
+      
+      next if !a.has?(:time)
+
+      unless a.get(:time) == time
+        return true
+      end
+      
+      a.unset(:time)
+      
+    end
+    
+    false
+    
+  end
+  
+end

data/lib/treat/extractors/topic_words/lda.rb

@@ -1,63 +1,72 @@
-module Treat
-  module Extractors
-    module TopicWords
-      # An adapter for the 'lda-ruby' gem, which clusters
-      # documents into topics based on Latent Dirichlet
-      # Allocation.
-      #
-      # Original paper:
-      # Blei, David M., Ng, Andrew Y., and Jordan, Michael
-      # I. 2003. Latent dirichlet allocation. Journal of
-      # Machine Learning Research. 3 (Mar. 2003), 993-1022.
-      #
-      # Project website: https://github.com/ealdent/lda-ruby
-      class LDA
-        # Require the lda-ruby gem.
-        silence_warnings { require 'lda-ruby' }
-        # Monkey patch the TextCorpus class to call it without
-        # having to create any files.
-        Lda::TextCorpus.class_eval do
-          # Ruby, Y U NO SHUT UP!
-          silence_warnings { undef :initialize }
-          # Redefine initialize to take in an array of sections
-          def initialize(sections)
-            super(nil)
-            sections.each do |section|
-              add_document(Lda::TextDocument.new(self, section))
-            end
-          end
-        end
-        # Default options for the LDA algorithm.
-        DefaultOptions = {
-          :num_topics => 20,
-          :words_per_topic => 10,
-          :iterations => 20
-        }
-        # Retrieve the topic words of a collection.
-        def self.topic_words(collection, options = {})
-          options = DefaultOptions.merge(options)
-          # Create a corpus with the collection
-          sections = collection.sections.collect do |t|
-            t.to_s.encode('UTF-8', :invalid => :replace,
-            :undef => :replace, :replace => "?")            # Fix
-          end
-          corpus = Lda::TextCorpus.new(sections)
+# An adapter for the 'lda-ruby' gem, which clusters
+# documents into topics based on Latent Dirichlet
+# Allocation.
+#
+# Original paper:
+# Blei, David M., Ng, Andrew Y., and Jordan, Michael
+# I. 2003. Latent dirichlet allocation. Journal of
+# Machine Learning Research. 3 (Mar. 2003), 993-1022.
+#
+# Project website: https://github.com/ealdent/lda-ruby
+module Treat::Extractors::TopicWords::LDA
 
-          # Create an Lda object for training
-          lda = Lda::Lda.new(corpus)
-          lda.num_topics = options[:num_topics]
-          lda.max_iter = options[:iterations]
-          # Run the EM algorithm using random starting points
-          silence_stdout { lda.em('random') }
-          # Load the vocabulary.
-          if options[:vocabulary]
-            lda.load_vocabulary(options[:vocabulary])
-          end
-
-          # Get the topic words.
-          lda.top_words(options[:words_per_topic])
-        end
+  # Require the lda-ruby gem.
+  silence_warnings { require 'lda-ruby' }
+  
+  # Monkey patch the TextCorpus class to 
+  # call it without having to create any files.
+  Lda::TextCorpus.class_eval do
+    # Ruby, Y U NO SHUT UP!
+    silence_warnings { undef :initialize }
+    # Redefine initialize to take in an 
+    # array of sections.
+    def initialize(sections)
+      super(nil)
+      sections.each do |section|
+        add_document(
+        Lda::TextDocument.new(self, section))
       end
     end
   end
+  
+  # Default options for the LDA algorithm.
+  DefaultOptions = {
+    :num_topics => 20,
+    :words_per_topic => 10,
+    :iterations => 20,
+    :vocabulary => nil
+  }
+  
+  # Retrieve the topic words of a collection.
+  def self.topic_words(collection, options = {})
+
+    options = DefaultOptions.merge(options)
+    
+    docs = collection.documents.map { |d| d.to_s }
+    # Create a corpus with the collection
+    corpus = Lda::TextCorpus.new(docs)
+    
+    # Create an Lda object for training
+    lda = Lda::Lda.new(corpus)
+    lda.num_topics = options[:num_topics]
+    lda.max_iter = options[:iterations]
+    # Run the EM algorithm using random 
+    # starting points
+    
+    silence_stdout do
+      lda.em('random')
+    end
+    
+    # Load the vocabulary.
+    if options[:vocabulary]
+      lda.load_vocabulary(options[:vocabulary])
+    end
+    
+    # Get the topic words.
+    lda.top_words(
+    options[:words_per_topic]
+    ).values
+    
+  end
+  
 end

data/lib/treat/extractors/topics/reuters.rb

@@ -1,92 +1,105 @@
-module Treat
-  module Extractors
-    module Topics
-      # A Ruby Part text categorizer that was trained
-      # using the Reuters news story corpus.  Version 0.1
-      #
-      # Copyright 2005 Mark Watson.  All rights reserved.
-      # This software is released under the GPL.
-      # Rewrite for inclusion in Treat by Louis Mullie (2011).
-      # 
-      # Original project website: http://www.markwatson.com/opensource/
-      class Reuters
-        # Require the Nokogiri XML parser.
-        require 'nokogiri'
-        # Hashes to hold the topics.
-        @@industry = {}
-        @@region = {}
-        @@topics = {}
-        # Get the topic of the text.
-        # 
-        # Options: none.
-        def self.topics(text, options = {})
-          stems = []
-          @@reduce = 0
-          unless text.words.size > 0
-            raise Treat::Exception,
-            "Annotator 'topics' requires processor 'tokenize'."
-          end
-          text.words.collect! do |tok|
-            stem = tok.stem.downcase
-            val = tok.value.downcase
-            stems << stem
-            unless stem == val
-              stems << val
-            end
-          end
-          get_topics
-          score_words(@@industry, stems) + 
-          score_words(@@region, stems) +
-          score_words(@@topics, stems)
-          #Treat::Feature.new(topics)
-        end
-        # Read the topics from the XML files.
-        def self.get_topics
-          return unless @@industry.size == 0
-          @@industry = read_xml(Treat.lib + '/treat/extractors/topics/reuters/industry.xml')
-          @@region = read_xml(Treat.lib + '/treat/extractors/topics/reuters/region.xml')
-          @@topics = read_xml(Treat.lib + '/treat/extractors/topics/reuters/topics.xml')
-        end
-        def self.read_xml(file_name)
-          hash = {}
-          doc = Nokogiri::XML(File.read(file_name))
-          doc.root.children.each do |category|
-            cat = category["cat"]
-            next if cat.nil?
-            cat = cat.downcase
-            hash[cat] ||= {}
-            hash[cat][category["name"]] =
-            category["score"].to_f
-          end
-          hash
-        end
-        def self.score_words(hash, word_list)
-          category_names = hash.keys
-          count_hash = {}
-          category_names.each do |cat_name|
-            cat_name = cat_name.downcase
-            count_hash[cat_name] ||= 0
-            word_list.each do |word|
-              unless hash[cat_name][word].nil?
-                count_hash[cat_name] += 
-                hash[cat_name][word]
-              end
-            end
-          end
-          count_hash = best_of_hash(count_hash)
-          count_hash.keys
-        end
-        def self.best_of_hash(hash, cutoff = 0.0, scale = 1.0)
-          ret = {}
-          hash.keys.each do |key|
-            if hash[key] > cutoff
-              ret[key] = hash[key] * scale 
-              ret[key] = ret[key].round(2)
-            end
-          end
-          ret
+# A Ruby text categorizer that was trained using 
+# the Reuters news story corpus. 
+#
+# Copyright 2005 Mark Watson. All rights reserved.
+# Rewrite for inclusion in Treat by Louis Mullie (2011).
+#
+# Original project website: 
+# http://www.markwatson.com/opensource/
+module Treat::Extractors::Topics::Reuters
+
+  # Require the Nokogiri XML parser.
+  require 'nokogiri'
+  
+  # Hashes to hold the topics.
+  @@industry = {}
+  @@region = {}
+  @@topics = {}
+  
+  # Get the general topic of the text using
+  # a Reuters-trained model.
+  #
+  # Options: none.
+  def self.topics(text, options = {})
+    stems = []
+    @@reduce = 0
+    unless text.words.size > 0
+      raise Treat::Exception,
+      "Annotator 'topics' requires " +
+      "processor 'tokenize'."
+    end
+    text.words.collect! do |tok|
+      stem = tok.stem.downcase
+      val = tok.value.downcase
+      stems << stem
+      unless stem == val
+        stems << val
+      end
+    end
+    get_topics
+    score_words(@@industry, stems) +
+    score_words(@@region, stems) +
+    score_words(@@topics, stems)
+    #Treat::Feature.new(topics)
+  end
+  
+  # Read the topics from the XML files.
+  def self.get_topics
+    return unless @@industry.size == 0
+    @@industry = read_xml(Treat.models + 
+    'reuters/industry.xml')
+    @@region = read_xml(Treat.models + 
+    'reuters/region.xml')
+    @@topics = read_xml(Treat.models + 
+    'reuters/topics.xml')
+  end
+  
+  # Read an XML file and populate a
+  # hash of topics.
+  def self.read_xml(file_name)
+    hash = {}
+    doc = Nokogiri::XML(File.read(file_name))
+    doc.root.children.each do |category|
+      cat = category["cat"]
+      next if cat.nil?
+      cat = cat.downcase
+      hash[cat] ||= {}
+      hash[cat][category["name"]] =
+      category["score"].to_f
+    end
+    hash
+  end
+  
+  # Score the words by adding the scores
+  # of each word occurence.
+  def self.score_words(hash, word_list)
+    category_names = hash.keys
+    count_hash = {}
+    category_names.each do |cat_name|
+      cat_name = cat_name.downcase
+      count_hash[cat_name] ||= 0
+      word_list.each do |word|
+        unless hash[cat_name][word].nil?
+          count_hash[cat_name] +=
+          hash[cat_name][word]
         end
       end
     end
+    count_hash = best_of_hash(count_hash)
+    count_hash.keys
+  end
+  
+  # Retrieve the words with the scores above
+  # cutoff inside the hash of scored words.
+  def self.best_of_hash(hash, cutoff = 0.0, scale = 1.0)
+    ret = {}
+    hash.keys.each do |key|
+      if hash[key] > cutoff
+        ret[key] = hash[key] * scale
+        ret[key] = ret[key].round(2)
+      end
+    end
+    ret
   end
+  
 end