treat 0.2.5 → 1.0.0

data/lib/treat/exception.rb

@@ -1,7 +1,7 @@
 module Treat
   # Custom exception class for the Treat toolkit.
   # Used to distinguish between errors raised by
-  # gems or Ruby from errors raised by the toolkit.
-  class Exception < ::Exception
-  end
-end
+  # gems/Ruby from errors raised by the toolkit.
+  class Exception < ::Exception; end
+  class InvalidInputException < Exception; end
+end

data/lib/treat/extractors.rb

@@ -1,82 +1,79 @@
-module Treat
-  # Extractors extract specific information out of texts.
-  module Extractors
-    # Detecs language.
-    module Language
-      extend Group
-      require 'treat/extractors/language/language_extractor.rb'
-      self.type = :annotator
-      self.targets = [:entity]
-      self.default = :what_language
-    end
-    # Extracts the time of an object and annotates it
-    # with specific information regarding time.
-    module Time
-      extend Group
-      self.type = :annotator
-      self.targets = [:phrase]
-    end
-    # Extracts the time of an object and annotates it
-    # with specific information regarding time.
-    module Date
-      extend Group
-      self.type = :annotator
-      self.targets = [:phrase]
-    end
-    # Extract the topic from a text.
-    module Topics
-      extend Group
-      self.type = :annotator
-      self.targets = [:document, :zone]
-    end
-    # Extract the keywords from a text.
-    module Keywords
-      extend Group
-      self.type = :annotator
-      self.targets = [:document, :zone]
-    end
-    # Extract the topic words from a text.
-    module TopicWords
-      extend Group
-      self.type = :annotator
-      self.targets = [:collection]
-    end
-    # Extract named entities from texts.
-    module NamedEntityTag
-      extend Group
-      self.type = :annotator
-      self.targets = [:phrase, :word]
-    end
-    # Extract named entities from texts.
-    module Coreferences
-      extend Group
-      self.type = :annotator
-      self.targets = [:zone]
-    end
-    # This module should be moved out of here ASAP.
-    module Statistics
-      extend Group
-      self.type = :annotator
-      self.targets = [:word]
-      self.default = :none
-      self.preprocessors = {
-        :frequency_in => lambda do |entity, worker, options|
-          options = {:parent => worker}.merge(options)
-          entity.statistics(:frequency_in, options)
-        end,
-        :tf_idf => lambda do |entity, worker, options|
-          entity.statistics(:tf_idf, options)
-        end,
-        :position_in => lambda do |entity, options|
-          entity.statistics(:position_in, options)
-        end
-      }
-    end
-    module Roles
-      extend Group
-      self.type = :annotator
-      self.targets = [:phrase]
-    end
-    extend Treat::Category
+# Extractors extract information out of texts.
+module Treat::Extractors
+
+  # Extracts the language from an entity.
+  module Language
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:entity]
+    self.default = :what_language
   end
+
+  # Extracts the date/time of a phrase.
+  module Time
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:phrase]
+  end
+
+  # Extract the topic from a document or zone.
+  module Topics
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:document]
+  end
+
+  # Extract the keywords from a text.
+  module Keywords
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:document]
+  end
+
+  # Extract clusters of topic words from a collection.
+  module TopicWords
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:collection]
+  end
+
+  # Extract named entities from phrases.
+  module NameTag
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:phrase, :word]
+  end
+
+  # Extract coreferences from a zone.
+  module Coreferences
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:zone]
+  end
+  
+  # Retrieve the main grammatical roles
+  # in the phrase (subject, verb, object).
+  module Roles
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:phrase]
+  end
+
+  module TfIdf
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:word]
+    self.default = :native
+  end
+  
+  module Summary
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:document]
+    self.default = :keyword_count
+  end
+  
+  # Make Extractors categorizable.
+  extend Treat::Categorizable
+  
 end

data/lib/treat/extractors/keywords/tf_idf.rb

@@ -1,26 +1,60 @@
-module Treat
-  module Extractors
-    module Keywords
-      class TfIdf
-        DefaultOptions = { num_keywords: 5 }
-        def self.keywords(entity, options = {})
-          options = DefaultOptions.merge(options)
-          tf_idfs = {}
-          entity.each_word do |word|
-            tf_idfs[word.value] ||= word.tf_idf
-          end
-          tf_idfs = tf_idfs.sort_by {|k,v| v}.reverse
-          return tf_idfs if tf_idfs.size <= options[:num_keywords]
-          keywords = []
-          i = 0
-          tf_idfs.each do |info|
-            break if i > options[:num_keywords]
-            keywords << info[0]
-            i += 1
-          end
-          keywords
+# This retrieves a supplied number of keywords
+# by selecting the N words with the highest TF*IDF
+# for each document.
+class Treat::Extractors::Keywords::TfIdf
+  
+  # Default options - retrieve 5 keywords.
+  DefaultOptions = { :number => 5 }
+  
+  # Annotate a document with an array containing
+  # the N words with the highest TF*IDF in that
+  # document,
+  def self.keywords(entity, options = {})
+    
+    options = DefaultOptions.merge(options)
+    tf_idfs = {}
+    entity.each_word do |word|
+      word.check_has(:tf_idf, false)
+      tf_idfs[word] ||= word.get(:tf_idf)
+    end
+
+    tf_idfs = tf_idfs.
+    sort_by {|k,v| v}.reverse
+
+    if tf_idfs.size <= options[:number]
+      return tf_idfs
+    end
+    
+    keywords = []
+    i = 0
+    
+    tf_idfs.each do |word|
+      
+      w = word[0].to_s
+      next if keywords.include?(w)
+
+      entity.each_word_with_value(w) do |w2|
+
+        ps = w2.parent_phrase
+        
+        if ps.has?(:keyword_count)
+          ps.set :keyword_count, 
+          ps.keyword_count + 1
+        else
+          ps.set :keyword_count, 1
         end
+        ps.set :keyword_density, 
+        (ps.keyword_count / ps.size)
+      
       end
+      
+      break if i > options[:number]
+      keywords << w
+      
+      i += 1
     end
+    
+    keywords
   end
-end
+  
+end

data/lib/treat/extractors/language/what_language.rb

@@ -1,49 +1,54 @@
-module Treat
-  module Extractors
-    module Language
-      # Require the 'whatlanguage' gem.
-      silence_warnings { require 'whatlanguage'  }
-      String.class_eval { undef :language }
-      DefaultOptions = {
-         :bias => [:eng, :fre, :chi, :ger, :ara, :spa]
-      }
-      # Adaptor for the 'whatlanguage' gem, which
-      # performs probabilistic language detection.
-      # The library works by checking for the presence 
-      # of words with bloom filters built from dictionaries 
-      # based upon each source language.
-      class WhatLanguage < LanguageExtractor
-        # Keep only once instance of the gem class.
-        @@detector = nil
-        # Detect the language of an entity using the
-        # 'whatlanguage' gem. Return an identifier
-        # corresponding to the ISO-639-2 code for the
-        # language.
-        #
-        # Options: 
-        # - (Array of Symbols) bias => Languages to bias 
-        # toward when more than one language is detected 
-        # with equal probability.
-        def self.language(entity, options = {})
-          options = DefaultOptions.merge(options)
-          predetection = super(entity, options)
-          return predetection if predetection
-          @@detector ||= ::WhatLanguage.new(:possibilities)
-          possibilities = @@detector.process_text(entity.to_s)
-          lang = {}
-          possibilities.each do |k,v|
-            lang[Treat::Languages.code(k)] = v
-          end
-          max = lang.values.max
-          ordered = lang.select { |i,j| j == max }.keys
-          ordered.each do |l|
-            if options[:bias].include?(l)
-              return l
-            end
-          end
-          return ordered.first
+module Treat::Extractors::Language
+  
+  # Adaptor for the 'whatlanguage' gem, which
+  # performs probabilistic language detection.
+  # The library works by checking for the presence
+  # of words with bloom filters built from 
+  # dictionaries based upon each source language.
+  class WhatLanguage
+    
+    # Require the 'whatlanguage' gem.
+    silence_warnings { require 'whatlanguage'  }
+    
+    # Undefine the method defined by the gem.
+    String.class_eval { undef :language }
+    
+    # By default, bias towards common languages.
+    DefaultOptions = {
+      :bias => [:eng, :fre, :chi, :ger, :ara, :spa]
+    }
+    
+    # Keep only once instance of the gem class.
+    @@detector = nil
+    
+    # Detect the language of an entity using the
+    # 'whatlanguage' gem. Return an identifier
+    # corresponding to the ISO-639-2 code for the
+    # language.
+    #
+    # Options:
+    #
+    # - (Array of Symbols) bias => Languages to bias
+    # toward when more than one language is detected
+    # with equal probability.
+    def self.language(entity, options = {})
+      options = DefaultOptions.merge(options)
+      @@detector ||= ::WhatLanguage.new(:possibilities)
+      possibilities = @@detector.process_text(entity.to_s)
+      lang = {}
+      possibilities.each do |k,v|
+        lang[Treat::Languages.code(k)] = v
+      end
+      max = lang.values.max
+      ordered = lang.select { |i,j| j == max }.keys
+      ordered.each do |l|
+        if options[:bias].include?(l)
+          return l
         end
       end
+      return ordered.first
     end
+    
   end
-end
+
+end

data/lib/treat/extractors/name_tag/stanford.rb

@@ -0,0 +1,55 @@
+# Detects the named entity tag in sentences by using
+# the stanford-core-nlp gem, which interfaces with
+# the Stanford Deterministic Coreference Resolver.
+class Treat::Extractors::NameTag::Stanford
+
+  require 'treat/loaders/stanford'
+
+  @@classifiers = {}
+
+  def self.name_tag(entity, options = {})
+
+    pp = nil
+
+    lang = entity.language
+
+    language = Treat::Languages.describe(lang)
+
+    isolated_token = entity.is_a?(Treat::Entities::Token)
+    tokens = isolated_token ? [entity] : entity.tokens
+
+    ms = StanfordCoreNLP::Config::Models[:ner][language]
+    ms = Treat.models + 'stanford/' +
+    StanfordCoreNLP::Config::ModelFolders[:ner] +
+    ms['3class']
+
+    @@classifiers[lang] ||=
+    StanfordCoreNLP::CRFClassifier.
+    getClassifier(ms)
+
+    token_list = StanfordCoreNLP.get_list(tokens)
+    sentence = @@classifiers[lang].classify_sentence(token_list)
+
+    i = 0
+    n = 0
+    
+    sentence.each do |s_token|
+      tag = s_token.get(:answer).to_s.downcase
+      tag = nil if tag == 'o'
+      return tag if isolated_token
+      if tag
+        tokens[i].set :name_tag, tag
+        n += 1
+      end
+      i += 1
+    end
+
+    entity.set :named_entity_count, n
+
+    nil
+    
+  end
+
+
+
+end

data/lib/treat/extractors/tf_idf/native.rb

@@ -0,0 +1,87 @@
+# Calculates the TF*IDF score of words.
+module Treat::Extractors::TfIdf::Native
+  DefaultOptions = {
+    :tf => :natural,
+    :idf => :logarithm,
+    :remove_common_words => true,
+    :precision => 4
+  }
+  Algorithms = {
+    :tf => {
+      :natural => lambda { |tf| tf },
+      :logarithm => lambda { |tf| Math.log(1 + tf) },
+      :sqrt =>lambda { |tf| Math.sqrt(tf) }
+    },
+    :idf => {
+      :logarithm => lambda { |n,df| Math.log(n/(1 + df)) },
+      :none => lambda { |n,idf| 1 }
+    }
+  }
+  # Optimization caches for tf idf.
+  @@n = {} # Number of documents in the collection (n).
+  @@df= {} # Number of documents that have a given value (document count).
+  @@f = {} # Number of times a word appears in a given document (term count).
+  @@wc = {} # Number of words in a given document (word count).
+  @@cw = {} # Common words to filter out.
+  def self.tf_idf(entity, options={})
+    l = Treat::Languages.get(entity.language)
+    if l.const_defined?(:CommonWords)
+      @@cw[entity.language] = 
+      l.const_get(:CommonWords)
+      return 0 if @@cw[entity.language].include?(entity.value)
+    end
+    return 0 if entity.value.length <= 2
+    options = DefaultOptions.merge(options)
+    lambdas = options.partition do |k,v|
+      [:tf, :idf, :normalization].include?(k)
+    end[0]
+    lambdas.each do |opt,val|
+      if opt.is_a?(Symbol)
+        if Algorithms[opt][val]
+          options[opt] = Algorithms[opt][val]
+        else
+          raise Treat::Exception,
+          "The specified algorithm '#{val}' "+
+          "to calculate #{opt} does not exist."
+        end
+      end
+    end
+    collection = entity.parent_collection
+    unless collection
+      raise Treat::Exception, "Cannot get the TF*IDF scores " +
+      "for a document that is not in a collection."
+    end
+    document = entity.parent_document
+    dc = collection.document_count
+    if !collection || !document
+      raise Treat::Exception,
+      "Tf*Idf requires a collection with documents."
+    end
+    val = entity.value.downcase
+    @@n[collection.id] = dc if @@n[collection.id].nil?
+    @@df[collection.id] ||= {}
+    if @@df[collection.id][val].nil?
+      df = 0
+      collection.each_document do |doc|
+        @@f[doc.id] ||= {}
+        if @@f[doc.id][val].nil?
+          @@f[doc.id][val] =
+          doc.frequency_of(val)
+        end
+        df += 1 if @@f[doc.id][val] > 0
+      end
+      @@df[collection.id][val] = df
+    end
+    f = @@f[document.id][entity.value].to_f
+    df = @@df[collection.id][entity.value].to_f
+    tf = options[:tf].call(f).to_f
+    if options[:normalize_word_count]
+      @@wc[document.id] ||= document.word_count
+      tf /= @@wc[document.id]
+    end
+    n = @@n[collection.id].to_f
+    idf = options[:idf].call(n, df)
+    tf_idf = tf * idf
+    tf_idf.abs.round(options[:precision])
+  end
+end