treat 0.2.5 → 1.0.0

data/lib/treat/doable.rb

@@ -1,45 +0,0 @@
-module Treat
-  module Doable
-    def do(*tasks)
-      tasks.each do |task|
-        if task.is_a?(Hash)
-          task.each do |k,v|
-            t, w = k, v
-            w, o = *w if w.is_a?(Array)
-            o ||= {}
-            do_task(t, w, o)
-          end
-        else
-          t = task.is_a?(Array) ? task[0] : task
-          w = task.is_a?(Array) ? task[1] : nil
-          w, o = *w if w.is_a?(Array)
-          o ||= {}
-          do_task(t, w, o)
-        end
-      end
-    end
-    DEBUG = true
-    def do_task(task, worker, options)
-      group = Categories.lookup(task)
-      unless group
-        raise Treat::Exception, "Task #{task} does not exist."
-      end
-      entity_types = group.targets
-      f = nil
-      entity_types.each do |t|
-        f = true if Treat::Entities.match_types[t][type]
-      end
-      if f || entity_types.include?(:entity)
-        send(task, worker, options)
-      else
-        each_entity(*entity_types) do |entity|
-          entity.do_task(task, worker, options)
-        end
-        unless entity_types.include?(type)
-          features.delete(task)
-        end
-        nil
-      end
-    end
-  end
-end

data/lib/treat/entities/collection.rb

@@ -1,14 +0,0 @@
-module Treat
-  module Entities
-    # Represents a collection of texts.
-    class Collection < Entity
-      # Initialize the collection with a folder
-      # containing the texts of the collection.
-      def initialize(folder = nil)
-        super('', id)
-        @type = :collection
-        set :folder, folder
-      end
-    end
-  end
-end

data/lib/treat/entities/document.rb

@@ -1,12 +0,0 @@
-module Treat
-  module Entities
-    # Represents a document.
-    class Document < Entity
-      def initialize(file = nil, id = nil)
-        super('', id)
-        set :file, file if file
-        @type = :document
-      end
-    end
-  end
-end

data/lib/treat/entities/phrases.rb

@@ -1,17 +0,0 @@
-module Treat
-  module Entities
-    # Represents any syntactic phrase of a sentence.
-    class Phrase < Entity
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :phrase
-      end
-    end
-    class Sentence < Phrase
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :sentence
-      end
-    end
-  end
-end

data/lib/treat/entities/tokens.rb

@@ -1,61 +0,0 @@
-module Treat
-  module Entities
-    # Represents a terminal element in the text structure.
-    class Token < Entity
-      # All tokens are leafs.
-      def is_leaf?; true; end
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :token
-      end
-    end
-    # Represents a word.
-    class Word < Token
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :word
-      end
-    end
-    # Represents a clitic ('s).
-    class Clitic < Token
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :clitic
-      end
-    end
-    # Represents a number.
-    class Number < Token
-      # Convert the number to an integer.
-      def to_i; to_s.to_i; end
-      # Convert the number to a float.
-      def to_f; to_s.to_f; end
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :number
-      end
-    end
-    # Represents a punctuation sign.
-    class Punctuation < Token
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :punctuation
-      end
-    end
-    # Represents a character that is neither
-    # alphabetical, numerical or a punctuation
-    # character (e.g. @#$%&*).
-    class Symbol < Token
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :symbol
-      end
-    end
-    # Represents an entity of unknown type.
-    class Unknown < Token
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :unknown
-      end
-    end
-  end
-end

data/lib/treat/entities/zones.rb

@@ -1,41 +0,0 @@
-module Treat
-  module Entities
-    # Represents a zone of text 
-    # (Title, Paragraph, List, Quote).
-    class Zone < Entity
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :zone
-      end
-    end
-    # Represents a title, subtitle, logical header.
-    class Title < Zone
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :title
-      end
-    end
-    # Represents a paragraph.
-    class Paragraph < Zone
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :paragraph
-      end
-    end
-    # Represents a list.
-    class List < Zone
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :list
-      end
-    end
-    # Represents a section, usually with a title
-    # and at least one paragraph.
-    class Section < Zone
-      def initialize(value = '', id = nil)
-        super(value, id)
-        @type = :section
-      end
-    end
-  end
-end

data/lib/treat/extractors/coreferences/stanford.rb

@@ -1,69 +0,0 @@
-module Treat
-  module Extractors
-    module Coreferences
-      class Stanford
-        require 'stanford-core-nlp'
-        @@pipeline = nil
-        def self.coreferences(entity, options = {})
-          val = entity.to_s
-          if entity.has_children?
-            warn "The Stanford Coreference Resolver currently requires " +
-            "an unsegmented, untokenized block of text to work with. " +
-            "Removing and replacing all children of '#{entity.short_value}'."
-            entity.remove_all!
-          end
-          @@pipeline ||=  ::StanfordCoreNLP.load(
-          :tokenize, :ssplit, :pos,
-          :lemma, :parse, :ner, :dcoref
-          )
-          text = ::StanfordCoreNLP::Text.new(entity.to_s)
-          @@pipeline.annotate(text)
-          clusters = {}
-          text.get(:sentences).each do |sentence|
-            s = Treat::Entities::Sentence.
-            from_string(sentence.get(:value).to_s, true)
-            sentence.get(:tokens).each do |token|
-              t = Treat::Entities::Token.
-              from_string(token.value.to_s)
-              tag = token.get(:named_entity_tag).
-              to_s.downcase
-              corefid = token.get(:coref_cluster_id).to_s
-              unless corefid == ''
-                clusters[corefid] ||= []
-                clusters[corefid] << t
-                t.set :coref_cluster_id, corefid
-              end
-
-              t.set :named_entity_tag,
-              tag.intern unless tag == 'o'
-              s << t
-            end
-            entity << s
-          end
-          entity.each_token do |token|
-            if token.has?(:coref_cluster_id)
-              id = token.coref_cluster_id
-              links = clusters[id].dup
-              links.delete(token)
-              token.unset(:coref_cluster_id)
-              next if links.empty?
-              token.set :coreferents, links
-              links.each do |target|
-                token.link(target, :refers_to)
-              end
-            end
-          end
-          i = 0
-          coreferences = {}
-          clusters.each do |k,v|
-            unless !v || v.size == 1
-              coreferences[i] = v
-              i += 1
-            end
-          end
-          coreferences
-        end
-      end
-    end
-  end
-end

data/lib/treat/extractors/date/chronic.rb

@@ -1,32 +0,0 @@
-module Treat
-  module Extractors
-    module Date
-      # A wrapper for the 'chronic' gem, which parses
-      # date information.
-      #
-      # Project website: http://chronic.rubyforge.org/
-      class Chronic
-        silence_warnings { require 'chronic' }
-        require 'date'
-        # Return the date information contained within the entity
-        # by parsing it with the 'chronic' gem.
-        #
-        # Options: none.
-        def self.date(entity, options = {})
-          date = nil
-          return if entity.has?(:time)
-          s = entity.to_s
-          s.gsub!('\/', '/')
-          s.strip!
-          silence_warnings do
-            date = ::Chronic.parse(s, {:guess => true})
-          end
-          entity.ancestors_with_type(:phrase).each do |a|
-            a.unset(:date) if a.has?(:date)
-          end
-          return date.to_date if date
-        end
-      end
-    end
-  end
-end

data/lib/treat/extractors/date/ruby.rb

@@ -1,25 +0,0 @@
-module Treat
-  module Extractors
-    module Date
-      # A wrapper for Ruby's native date parsing.
-      class Ruby
-        require 'date'
-        # Return a DateTime object representing the date/date
-        # contained within the entity, using Ruby's native
-        # date/date parser.
-        #
-        # Options: none.
-        def self.date(entity, options = {})
-          begin
-            s = entity.to_s.strip
-            s.gsub!('\/', '/')
-            date = ::DateTime.parse(s)
-            date.to_date
-          rescue
-            nil
-          end
-        end
-      end
-    end
-  end
-end

data/lib/treat/extractors/keywords/topics_tf_idf.rb

@@ -1,48 +0,0 @@
-module Treat
-  module Extractors
-    module Keywords
-      class TopicsTfIdf
-        DefaultOptions = {num_keywords: 5, tf_idf_threshold: 0.5, topic_words: nil}
-        def self.keywords(entity, options = {})
-          options = DefaultOptions.merge(options)
-          unless options[:topic_words]
-            options[:topic_words] = entity.parent_collection.topic_words
-          end
-          if Treat::Entities.rank(entity.type) <
-            Treat::Entities.rank(:sentence)
-            raise Treat::Exception, 'Cannot get the key ' +
-            'sentences of an entity smaller than a sentence.'
-          else
-            find_keywords(entity, options)
-          end
-        end
-        def self.find_keywords(entity, options)
-          keywords = []
-          entity.each_word do |word|
-            found = false
-            tf_idf = word.tf_idf
-            options[:topic_words].each do |i, topic_words|
-              next if keywords.include?(word.value)
-              if topic_words.include?(word.value)
-                found = true
-                if tf_idf > options[:tf_idf_threshold]
-                  keywords << word.value
-                  word.set :is_keyword?, found
-                end
-              end
-            end
-          end
-          i = 0
-          # Take a slice of keywords with i elements.
-          selected_keywords = []
-          keywords.each do |keyword|
-            break if i > options[:num_keywords]
-            selected_keywords << keyword
-            i += 1
-          end
-          selected_keywords
-        end
-      end
-    end
-  end
-end

data/lib/treat/extractors/language/language_extractor.rb

@@ -1,27 +0,0 @@
-module Treat
-  module Extractors
-    module Language
-      # A generic language detector, which is called before
-      # any language detector and ensures that configuration
-      # options concerning language are enforced (e.g. returns
-      # the default language when Treat.detect_language is false).
-      class LanguageExtractor
-        def self.language(entity, options = {})
-          if entity.to_s =~ /^[[:digit:]]+$/
-            return Treat.default_language
-          end
-          if Treat.detect_language == false
-            return Treat.default_language
-          else
-            dlvl = Treat.language_detection_level
-            if (Entities.rank(entity.type) < Entities.rank(dlvl)) &&
-               entity.has_parent?
-               anc = entity.ancestor_with_type(dlvl)
-               return anc.language if anc
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/treat/extractors/named_entity_tag/stanford.rb

@@ -1,53 +0,0 @@
-module Treat
-  module Extractors
-    module NamedEntityTag
-      class Stanford
-        require 'stanford-core-nlp'
-        StanfordCoreNLP.load_class('ArrayList', 'java.util')
-        StanfordCoreNLP.load_class('Word', 'edu.stanford.nlp.ling')
-        @@pipeline = nil
-        def self.named_entity_tag(entity, options = {})
-          pp = nil
-          if entity.is_a?(Treat::Entities::Token) && 
-             entity.has_parent?
-              pp = entity.parent_phrase
-              s = get_list(pp.tokens)
-          else
-            s = entity.to_s
-          end
-          
-          @@pipeline ||=  ::StanfordCoreNLP.load(
-          :tokenize, :ssplit, :pos, :lemma, :parse, :ner
-          )
-          
-          text = ::StanfordCoreNLP::Text.new(s)
-          @@pipeline.annotate(text)
-          
-          add_to = pp ? pp : entity
-          
-          if entity.is_a?(Treat::Entities::Phrase)
-            text.get(:tokens).each do |token|
-              t = Treat::Entities::Token.from_string(token.value.to_s)
-              tag = token.get(:named_entity_tag).to_s.downcase
-              t.set :named_entity_tag, tag.intern unless tag == 'o'
-              add_to << t
-            end
-          elsif entity.is_a?(Treat::Entities::Token)
-            tag = text.get(:tokens).iterator.next.
-            get(:named_entity_tag).to_s.downcase
-            entity.set :named_entity_tag, tag.intern unless tag == 'o'
-          end
-          
-        end
-
-        def self.get_list(words)
-          list = StanfordCoreNLP::ArrayList.new
-          words.each do |w|
-            list.add(StanfordCoreNLP::Word.new(w.to_s))
-          end
-          list
-        end
-      end
-    end
-  end
-end