treat 0.1.4 → 0.2.0

data/lib/treat/extractors.rb

@@ -1,43 +1,81 @@
 module Treat
   # Extractors extract specific information out of texts.
   module Extractors
+    # Detecs language.
+    module Language
+      extend Group
+      require 'treat/extractors/language/language_extractor.rb'
+      self.type = :annotator
+      self.targets = [:entity]
+      self.default = :what_language
+    end
     # Extracts the time of an object and annotates it
     # with specific information regarding time.
     module Time
       extend Group
       self.type = :annotator
-      self.targets = [:sentence, :word, :constituent, :symbol]
+      self.targets = [:phrase]
+    end
+    # Extracts the time of an object and annotates it
+    # with specific information regarding time.
+    module Date
+      extend Group
+      self.type = :annotator
+      self.targets = [:phrase]
     end
     # Extract the topic from a text.
     module Topics
       extend Group
       self.type = :annotator
-      self.targets = [:collection, :document, :zone, :sentence]
+      self.targets = [:document, :zone]
     end
-    # Extract the topic from a text.
+    # Extract the keywords from a text.
+    module Keywords
+      extend Group
+      self.type = :annotator
+      self.targets = [:document, :zone]
+    end
+    # Extract the topic words from a text.
     module TopicWords
       extend Group
       self.type = :annotator
-      self.targets = [:collection, :document, :zone, :sentence]
+      self.targets = [:collection]
     end
     # Extract named entities from texts.
-    module NamedEntity
+    module NamedEntityTag
       extend Group
-      self.type = :computer
-      self.targets = [:entity]
+      self.type = :annotator
+      self.targets = [:phrase, :word]
     end
-    # Extract the key sentences from a text.
-    module Keywords
+    # Extract named entities from texts.
+    module Coreferences
       extend Group
       self.type = :annotator
-      self.targets = [:collection, :document, :zone, :sentence]
+      self.targets = [:zone]
     end
     # This module should be moved out of here ASAP.
     module Statistics
       extend Group
       self.type = :annotator
-      self.targets = [:entity]
+      self.targets = [:word]
       self.default = :none
+      self.preprocessors = {
+        :frequency_in => lambda do |entity, worker, options|
+          options = {:parent => worker}.merge(options)
+          entity.statistics(:frequency_in, options)
+        end,
+        :tf_idf => lambda do |entity, worker, options|
+          entity.statistics(:tf_idf, options)
+        end,
+        :position_in => lambda do |entity, options|
+          entity.statistics(:position_in, options)
+        end
+      }
+    end
+    module Roles
+      extend Group
+      self.type = :annotator
+      self.targets = [:phrase]
     end
     extend Treat::Category
   end

data/lib/treat/extractors/coreferences/stanford.rb

@@ -0,0 +1,68 @@
+module Treat
+  module Extractors
+    module Coreferences
+      class Stanford
+        require 'stanford-core-nlp'
+        @@pipeline = nil
+        def self.coreferences(entity, options = {})
+          if entity.has_children?
+            warn "The Stanford Coreference Resolver currently requires " +
+            "an unsegmented, untokenized block of text to work with. " +
+            "Removing and replacing all children of '#{entity.short_value}'."
+            entity.remove_all!
+          end
+          @@pipeline ||=  ::StanfordCoreNLP.load(
+          :tokenize, :ssplit, :pos,
+          :lemma, :parse, :ner, :dcoref
+          )
+          text = ::StanfordCoreNLP::Text.new(entity.to_s)
+          @@pipeline.annotate(text)
+          clusters = {}
+          text.get(:sentences).each do |sentence|
+            s = Treat::Entities::Sentence.
+            from_string(sentence.get(:value).to_s, true)
+            sentence.get(:tokens).each do |token|
+              t = Treat::Entities::Token.
+              from_string(token.value.to_s)
+              tag = token.get(:named_entity_tag).
+              to_s.downcase
+              corefid = token.get(:coref_cluster_id).to_s
+              unless corefid == ''
+                clusters[corefid] ||= []
+                clusters[corefid] << t
+                t.set :coref_cluster_id, corefid
+              end
+
+              t.set :named_entity_tag,
+              tag.intern unless tag == 'o'
+              s << t
+            end
+            entity << s
+          end
+          entity.each_token do |token|
+            if token.has?(:coref_cluster_id)
+              id = token.coref_cluster_id
+              links = clusters[id].dup
+              links.delete(token)
+              token.unset(:coref_cluster_id)
+              next if links.empty?
+              token.set :coreferents, links
+              links.each do |target|
+                token.link(target, :refers_to)
+              end
+            end
+          end
+          i = 0
+          coreferences = {}
+          clusters.each do |k,v|
+            unless !v || v.size == 1
+              coreferences[i] = v
+              i += 1
+            end
+          end
+          coreferences
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/date/chronic.rb

@@ -0,0 +1,32 @@
+module Treat
+  module Extractors
+    module Date
+      # A wrapper for the 'chronic' gem, which parses
+      # date information.
+      #
+      # Project website: http://chronic.rubyforge.org/
+      class Chronic
+        silence_warnings { require 'chronic' }
+        require 'date'
+        # Return the date information contained within the entity
+        # by parsing it with the 'chronic' gem.
+        #
+        # Options: none.
+        def self.date(entity, options = {})
+          date = nil
+          return if entity.has?(:time)
+          s = entity.to_s
+          s.gsub!('\/', '/')
+          s.strip!
+          silence_warnings do
+            date = ::Chronic.parse(s, {:guess => true})
+          end
+          entity.ancestors_with_type(:phrase).each do |a|
+            a.unset(:date) if a.has?(:date)
+          end
+          return date.to_date if date
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/date/ruby.rb

@@ -0,0 +1,25 @@
+module Treat
+  module Extractors
+    module Date
+      # A wrapper for Ruby's native date parsing.
+      class Ruby
+        require 'date'
+        # Return a DateTime object representing the date/date
+        # contained within the entity, using Ruby's native
+        # date/date parser.
+        #
+        # Options: none.
+        def self.date(entity, options = {})
+          begin
+            s = entity.to_s.strip
+            s.gsub!('\/', '/')
+            date = ::DateTime.parse(s)
+            date.to_date
+          rescue
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/keywords/tf_idf.rb

@@ -0,0 +1,26 @@
+module Treat
+  module Extractors
+    module Keywords
+      class TfIdf
+        DefaultOptions = { num_keywords: 5 }
+        def self.keywords(entity, options = {})
+          options = DefaultOptions.merge(options)
+          tf_idfs = {}
+          entity.each_word do |word|
+            tf_idfs[word.value] ||= word.tf_idf
+          end
+          tf_idfs = tf_idfs.sort_by {|k,v| v}.reverse
+          return tf_idfs if tf_idfs.size <= options[:num_keywords]
+          keywords = []
+          i = 0
+          tf_idfs.each do |info|
+            break if i > options[:num_keywords]
+            keywords << info[0]
+            i += 1
+          end
+          keywords
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/keywords/{topics_frequency.rb → topics_tf_idf.rb}

@@ -1,12 +1,12 @@
 module Treat
   module Extractors
     module Keywords
-      class TopicsFrequency
-        DefaultOptions = {tf_idf_threshold: 180, topic_words: nil}
+      class TopicsTfIdf
+        DefaultOptions = {num_keywords: 5, tf_idf_threshold: 0.5, topic_words: nil}
         def self.keywords(entity, options = {})
           options = DefaultOptions.merge(options)
           unless options[:topic_words]
-            raise Treat::Exception, "You must supply topic words."
+            options[:topic_words] = entity.parent_collection.topic_words
           end
           if Treat::Entities.rank(entity.type) <
             Treat::Entities.rank(:sentence)
@@ -20,21 +20,29 @@ module Treat
           keywords = []
           entity.each_word do |word|
             found = false
+            tf_idf = word.tf_idf
             options[:topic_words].each do |i, topic_words|
               next if keywords.include?(word.value)
               if topic_words.include?(word.value)
                 found = true
-                tf_idf = word.tf_idf
-                if tf_idf < options[:tf_idf_threshold]
+                if tf_idf > options[:tf_idf_threshold]
                   keywords << word.value
                   word.set :is_keyword?, found
                 end
               end
             end
           end
-          keywords
+          i = 0
+          # Take a slice of keywords with i elements.
+          selected_keywords = []
+          keywords.each do |keyword|
+            break if i > options[:num_keywords]
+            selected_keywords << keyword
+            i += 1
+          end
+          selected_keywords
         end
       end
     end
   end
-end
+end

data/lib/treat/{detectors/language/language_detector.rb → extractors/language/language_extractor.rb}

@@ -1,12 +1,15 @@
 module Treat
-  module Detectors
+  module Extractors
     module Language
       # A generic language detector, which is called before
       # any language detector and ensures that configuration
       # options concerning language are enforced (e.g. returns
       # the default language when Treat.detect_language is false).
-      class LanguageDetector
+      class LanguageExtractor
         def self.language(entity, options = {})
+          if entity.to_s =~ /^[[:digit:]]+$/
+            return Treat.default_language
+          end
           if Treat.detect_language == false
             return Treat.default_language
           else

data/lib/treat/extractors/language/what_language.rb

@@ -0,0 +1,49 @@
+module Treat
+  module Extractors
+    module Language
+      # Require the 'whatlanguage' gem.
+      silence_warnings { require 'whatlanguage'  }
+      String.class_eval { undef :language }
+      DefaultOptions = {
+         :bias => [:eng, :fre, :chi, :ger, :ara, :spa]
+      }
+      # Adaptor for the 'whatlanguage' gem, which
+      # performs probabilistic language detection.
+      # The library works by checking for the presence 
+      # of words with bloom filters built from dictionaries 
+      # based upon each source language.
+      class WhatLanguage < LanguageExtractor
+        # Keep only once instance of the gem class.
+        @@detector = nil
+        # Detect the language of an entity using the
+        # 'whatlanguage' gem. Return an identifier
+        # corresponding to the ISO-639-2 code for the
+        # language.
+        #
+        # Options: 
+        # - (Array of Symbols) bias => Languages to bias 
+        # toward when more than one language is detected 
+        # with equal probability.
+        def self.language(entity, options = {})
+          options = DefaultOptions.merge(options)
+          predetection = super(entity, options)
+          return predetection if predetection
+          @@detector ||= ::WhatLanguage.new(:possibilities)
+          possibilities = @@detector.process_text(entity.to_s)
+          lang = {}
+          possibilities.each do |k,v|
+            lang[Treat::Languages.code(k)] = v
+          end
+          max = lang.values.max
+          ordered = lang.select { |i,j| j == max }.keys
+          ordered.each do |l|
+            if options[:bias].include?(l)
+              return l
+            end
+          end
+          return ordered.first
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/named_entity_tag/stanford.rb

@@ -0,0 +1,53 @@
+module Treat
+  module Extractors
+    module NamedEntityTag
+      class Stanford
+        require 'stanford-core-nlp'
+        StanfordCoreNLP.load_class('ArrayList', 'java.util')
+        StanfordCoreNLP.load_class('Word', 'edu.stanford.nlp.ling')
+        @@pipeline = nil
+        def self.named_entity_tag(entity, options = {})
+          pp = nil
+          if entity.is_a?(Treat::Entities::Token) && 
+             entity.has_parent?
+              pp = entity.parent_phrase
+              s = get_list(pp.tokens)
+          else
+            s = entity.to_s
+          end
+          
+          @@pipeline ||=  ::StanfordCoreNLP.load(
+          :tokenize, :ssplit, :pos, :lemma, :parse, :ner
+          )
+          
+          text = ::StanfordCoreNLP::Text.new(s)
+          @@pipeline.annotate(text)
+          
+          add_to = pp ? pp : entity
+          
+          if entity.is_a?(Treat::Entities::Phrase)
+            text.get(:tokens).each do |token|
+              t = Treat::Entities::Token.from_string(token.value.to_s)
+              tag = token.get(:named_entity_tag).to_s.downcase
+              t.set :named_entity_tag, tag.intern unless tag == 'o'
+              add_to << t
+            end
+          elsif entity.is_a?(Treat::Entities::Token)
+            tag = text.get(:tokens).iterator.next.
+            get(:named_entity_tag).to_s.downcase
+            entity.set :named_entity_tag, tag.intern unless tag == 'o'
+          end
+          
+        end
+
+        def self.get_list(words)
+          list = StanfordCoreNLP::ArrayList.new
+          words.each do |w|
+            list.add(StanfordCoreNLP::Word.new(w.to_s))
+          end
+          list
+        end
+      end
+    end
+  end
+end

data/lib/treat/extractors/roles/naive.rb

@@ -0,0 +1,73 @@
+module Treat
+  module Extractors
+    module Roles
+      class Naive
+        def self.roles(entity, options = {})
+          v = main_verb(entity, options)
+          return Treat::Features::Roles.new unless (v && v.has?(:voice))
+          o = object(v, options)
+          s = subject(v, options)
+          if v.voice == 'active'
+            p = o
+          elsif v.voice == 'passive'
+            p = s
+          elsif v.has_feature?(:aux)
+            p = s
+          end
+          p.set :is_patient?, true if p
+          if v.voice == 'active'
+            a = s
+          elsif v.voice == 'passive'
+            #a = object(entity, options)
+          end
+          a.set :is_agent?, true if a
+          if a && p
+            a.link(p, :agent_of)
+            p.link(a, :patient_of)
+          end
+          # Fix - s, o, v
+          Treat::Features::Roles.new(s, o, v, p, a)
+        end
+        # Return the subject of the sentence|verb.
+        def self.subject(verb, options)
+          args = []
+          return unless verb
+          verb.dependencies.each do |dependency|
+            args << verb.root.find(dependency.target)
+          end
+          s = args[0]
+          s.set :is_subject?, true if s
+          s
+        end
+        # Return the object of the sentence|verb.
+        def self.object(verb, options)
+          return if verb.has?(:voice) && verb.voice == 'passive'
+          args = []
+          verb.dependencies.each do |dependency|
+            args << verb.root.find(dependency.target)
+          end
+          o = args[1]
+          return unless o
+          if o.tag == 'NP'
+            b = o
+          else
+            b = o.phrases_with_tag('NP')[0]
+          end
+          b.set :is_object?, true if b
+          b
+        end
+        # Find the main verb (shallowest verb in the tree).
+        def self.main_verb(entity, options)
+          verbs = entity.verbs
+          if verbs.size == 0
+            return
+          end
+          verbs.sort! { |a,b| a.depth <=> b.depth }
+          v = verbs[0]
+          v.set :is_main_verb?, true if v
+          v
+        end
+      end
+    end
+  end
+end