treat 0.2.5 → 1.0.0

data/lib/treat/processors/tokenizers/macintyre.rb

@@ -1,77 +0,0 @@
-module Treat
-  module Processors
-    module Tokenizers
-      # A native rule-basd tokenizer based on the one
-      # developped by Robert Macyntyre in 1995 for the Penn
-      # Treebank project. This tokenizer follows the 
-      # conventions used by the Penn Treebank.
-      # 
-      # Original script: 
-      # http://www.cis.upenn.edu/~treebank/tokenizer.sed
-      #
-      # Copyright (c) 2004 UTIYAMA Masao <mutiyama@nict.go.jp>
-      # All rights reserved. This program is free software; 
-      # you can redistribute it and/or modify it under the 
-      # same terms as Ruby itself.
-      class Macintyre
-        # Tokenize the entity using a native rule-based algorithm.
-        def self.tokenize(entity, options = {})
-          if entity.has_children?
-            raise Treat::Exception,
-              'Cannot tokenize a Phrase that already has children.'
-          end
-          chunks = split(entity.to_s)
-          chunks.each do |chunk|
-            next if chunk =~ /([[:space:]]+)/
-            entity << Treat::Entities::Token.from_string(chunk)
-          end 
-        end
-        # Helper method to split the string into tokens.
-        def self.split(string)
-          s = " " + string + " "
-          s.gsub!(/\s+/," ")
-          s.gsub!(/(\s+)''/,'\1"')
-          s.gsub!(/(\s+)``/,'\1"')
-          s.gsub!(/''(\s+)/,'"\1')
-          s.gsub!(/``(\s+)/,'"\1')
-          s.gsub!(/ (['`]+)([^0-9].+) /,' \1 \2 ')
-          s.gsub!(/([ (\[{<])"/,'\1 `` ')
-          s.gsub!(/\.\.\./,' ... ')
-          s.gsub!(/[,;:@\#$%&]/,' \& ')
-          s.gsub!(/([^.])([.])([\])}>"']*)[ 	]*$/,'\1 \2\3 ')
-          s.gsub!(/[?!]/,' \& ')
-          s.gsub!(/[\]\[(){}<>]/,' \& ')
-          s.gsub!(/--/,' -- ')
-          s.sub!(/$/,' ')
-          s.sub!(/^/,' ')
-          s.gsub!(/"/,' \'\' ')
-          s.gsub!(/([^'])' /,'\1 \' ')
-          s.gsub!(/'([sSmMdD]) /,' \'\1 ')
-          s.gsub!(/'ll /,' \'ll ')
-          s.gsub!(/'re /,' \'re ')
-          s.gsub!(/'ve /,' \'ve ')
-          s.gsub!(/n't /,' n\'t ')
-          s.gsub!(/'LL /,' \'LL ')
-          s.gsub!(/'RE /,' \'RE ')
-          s.gsub!(/'VE /,' \'VE ')
-          s.gsub!(/N'T /,' N\'T ')
-          s.gsub!(/ ([Cc])annot /,' \1an not ')
-          s.gsub!(/ ([Dd])'ye /,' \1\' ye ')
-          s.gsub!(/ ([Gg])imme /,' \1im me ')
-          s.gsub!(/ ([Gg])onna /,' \1on na ')
-          s.gsub!(/ ([Gg])otta /,' \1ot ta ')
-          s.gsub!(/ ([Ll])emme /,' \1em me ')
-          s.gsub!(/ ([Mm])ore'n /,' \1ore \'n ')
-          s.gsub!(/ '([Tt])is /,' \'\1 is ')
-          s.gsub!(/ '([Tt])was /,' \'\1 was ')
-          s.gsub!(/ ([Ww])anna /,' \1an na ')
-          while s.sub!(/(\s)([0-9]+) , ([0-9]+)(\s)/, '\1\2,\3\4'); end
-          s.gsub!(/\//, ' / ')
-          s.gsub!(/\s+/,' ')
-          s.strip!
-          s.split(/\s+/)
-        end
-      end
-    end
-  end
-end

data/lib/treat/processors/tokenizers/multilingual.rb

@@ -1,30 +0,0 @@
-module Treat
-  module Processors
-    module Tokenizers
-      # An adapter for the 'tokenizer' gem, which performs
-      # rule-based tokenizing of texts in English, German
-      # or French.
-      class Multilingual
-        # Hold one tokenizer per language.
-        @@tokenizers = {}
-        # Require the 'tokenizer' gem.
-        silence_warnings { require 'tokenizer' }
-        # Perform the tokenization of English, German or French text.
-        # Options:
-        # :language => (Symbol) Force a language for the tokenizer.
-        def self.tokenize(entity, options = {})
-          lang = options[:language] ? options[:language] : entity.language
-          lang = Treat::Languages.code(lang, 1)    
-          if @@tokenizers[lang].nil?
-            @@tokenizers[lang] = ::Tokenizer::Tokenizer.new(lang)
-          end
-          tokens = @@tokenizers[lang].tokenize(entity.to_s)
-          tokens.each do |token|
-            next if token =~ /([[:space:]]+)/
-            entity << Treat::Entities::Token.from_string(token)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/treat/registrable.rb

@@ -1,28 +0,0 @@
-module Treat
-  module Registrable
-    # Registers a token in the @token_registry hash.
-    def register_token(token)
-      @token_registry ||= {:value => {}, :id => {}}
-      @token_registry[:id][token.id] = token
-      v = token.to_s.downcase
-      @token_registry[:value][v] ||= []
-      @token_registry[:value][v] << token
-      @parent.register_token(token) if has_parent?
-    end
-    # Find the token registry, by default the one
-    # in the root node.
-    def token_registry(type = nil)
-      if (type == nil && is_root?) || type == self.type
-        @token_registry ||= {:value => {}, :id => {}}
-        return @token_registry
-      else
-        if has_parent?
-          @parent.token_registry(type)
-        else
-          @token_registry ||= {:value => {}, :id => {}}
-          @token_registry
-        end
-      end
-    end
-  end
-end

data/lib/treat/sugar.rb

@@ -1,50 +0,0 @@
-module Treat
-  # This module provides syntactic sugar in the following manner:
-  # all entities found under Treat::Entities will be made
-  # available within the global namespace. For example,
-  # Treat::Entities::Word can now be referred to as simply 'Word'.
-  module Sugar
-    # Installs syntactic sugar.
-    def sweeten!
-      return if @@sweetened
-      @@sweetened = true
-      each_entity_class do |type, klass|
-        unless type == :Symbol
-          Object.class_eval do
-            define_method(type) do |value='',id=nil|
-              klass.build(value, id)
-            end
-          end
-        end
-      end
-    end
-
-    # Uninstalls syntactic sugar.
-    def unsweeten!
-      return unless @@sweetened
-      @@sweetened = false
-      each_entity_class do |type, klass| 
-        unless type == :Symbol
-          Object.class_eval do
-             remove_method(type)
-          end
-        end
-      end
-    end
-
-    # Boolean - whether syntactic sugar is
-    # enabled or not.
-    def sweetened?; @@sweetened; end
-    # Syntactic sugar is disabled by default.
-    @@sweetened = false
-    private
-    # Helper method, yields each entity type and class.
-    def each_entity_class
-      Treat::Entities.list.each do |entity_type|
-        type = cc(entity_type).intern
-        klass = Treat::Entities.const_get(type, klass)
-        yield type, klass
-      end
-    end
-  end
-end

data/lib/treat/viewable.rb

@@ -1,29 +0,0 @@
-module Treat
-  module Viewable
-    # Return the entity's string value in plain text format.
-    def to_string; @value; end
-    # An alias for #to_string.
-    def to_s; visualize(:txt); end
-    alias :to_str :to_s
-    # Return a shortened value of the entity's string value using [...].
-    def short_value(ml = 6); visualize(:short_value, :max_length => ml); end
-    # Return an informative string representation of the entity.
-    def inspect
-      s = "#{cl(self.class)} (#{@id.to_s})"
-      if caller_method(2) == :inspect
-        @id.to_s
-      else
-        dependencies = []
-        @dependencies.each do |dependency|
-          dependencies << "#{dependency.target}#{dependency.type}"
-        end
-        s += "  |  #{short_value.inspect}" +
-        "  |  #{@features.inspect}" +
-        "  | { #{dependencies.join(', ')} }"
-      end
-      s
-    end
-    # Print out an ASCII representation of the tree.
-    def print_tree; puts visualize(:tree); end
-  end
-end

data/lib/treat/visitable.rb

@@ -1,28 +0,0 @@
-module Treat
-  # Make a tree visitable by implementing the method #accept.
-  module Visitable
-    # Accept a visitor implemented by klass, which is
-    # found in the supplied group, and call method on it.
-    def accept(group, klass, method, options)
-      if group.has_target?(self.class)
-        if group.type == :transformer
-          if has_children?
-            @children.each do |entity|
-              if group.has_target?(entity.class) && entity.id != id
-                entity.accept(group, klass, method, options)
-              end
-            end
-          else
-            klass.send(method, self, options)
-          end
-          return self
-        else
-          return klass.send(method, self, options)
-        end
-      else
-        raise Treat::Exception,
-        "This type of visitor cannot visit a #{self.class}."
-      end
-    end
-  end
-end

data/test/profile.rb

@@ -1,2 +0,0 @@
-require 'unprof'
-require 'tests'

data/test/tc_entity.rb

@@ -1,117 +0,0 @@
-module Treat
-  module Tests
-    class TestEntity < Test::Unit::TestCase
-      def setup
-        @section = Treat::Entities::Section.new
-        @sentence = Treat::Entities::Sentence.new
-        @noun_cons = Treat::Entities::Phrase.new
-        @noun_cons.set :tag, 'NP'
-        @verb_cons = Treat::Entities::Phrase.new
-        @verb_cons.set :tag, 'VP'
-        @adj_cons = Treat::Entities::Phrase.new
-        @adj_cons.set :tag, 'ADJP'
-        @det = Treat::Entities::Word.new('The')
-        @det.set :category, :determiner
-        @det.set :tag, 'DT'
-        @det.set :tag_set, :penn
-        @adj = Treat::Entities::Word.new('lazy')
-        @adj.set :category, :adjective
-        @adj.set :tag, 'JJ'
-        @adj.set :tag_set, :penn
-        @noun = Treat::Entities::Word.new('fox')
-        @noun.set :category, :noun
-        @noun.set :tag, 'NN'
-        @noun.set :tag_set, :penn
-        @aux = Treat::Entities::Word.new('is')
-        @aux.set :category, :verb
-        @aux.set :tag, 'VBZ'
-        @aux.set :tag_set, :penn
-        @verb = Treat::Entities::Word.new('running')
-        @verb.set :category, :verb
-        @verb.set :tag, 'VBG'
-        @verb.set :tag_set, :penn
-        @dot = Treat::Entities::Punctuation.new('.')
-        @section << @sentence << [@noun_cons, @verb_cons, @dot]
-        @noun_cons << [@det, @adj_cons, @noun]
-        @adj_cons << @adj
-        @verb_cons << [@aux, @verb]
-      end
-
-      def test_viewable
-        s = 'Happiness is not an ideal of reason, but of imagination.'.tokenize
-        assert_nothing_raised do
-          # Return the string value of the sentence.
-          s.to_s
-          # Return a debug description of the sentence.
-          s.inspect
-          # Return a shortened version of the Sentence with [...]
-          s.short_value
-        end
-      end
-
-      def test_registrable
-        assert_equal @section.token_registry, @verb.token_registry
-        assert_equal @noun, @section.token_registry[:id][@noun.id]
-        assert_equal [@noun], @section.token_registry[:value][@noun.value]
-      end
-
-      def test_delegatable_visitable
-        assert_raise(Treat::Exception) do
-          @section.encoding(:nonexistent)
-        end
-        assert_nothing_raised do
-          @section.language
-        end
-      end
-
-      def test_type
-        assert_equal :section, @section.type
-      end
-
-      def test_printers
-        assert_nothing_raised do
-          @section.to_s
-          @section.to_string
-          @section.short_value
-          @section.inspect
-        end
-      end
-
-      def test_magic_methods
-
-        assert_equal true, @sentence.is_sentence?
-        assert_equal true, @noun.is_noun?
-
-        assert_equal @sentence, @section.sentence
-        assert_equal [@sentence], @section.sentences
-        assert_equal 1, @section.sentence_count
-
-        assert_equal [@det], @section.words_with_value('The')
-        assert_equal [@verb], @section.words_with_tag('VBG')
-
-        assert_equal @noun, @section.noun
-        assert_equal [@aux, @verb], @section.verbs
-        assert_equal 6, @section.token_count
-
-        @section.each_sentence do |s|
-          assert_equal @sentence, s
-        end
-        @section.each_noun do |n|
-          assert_equal @noun, n
-        end
-        @section.each_with_value('The') do |x|
-          assert_equal @det, x
-        end
-
-        assert_equal @sentence, @noun.parent_sentence
-      end
-
-      def test_features
-        @verb.set :test, :test
-        assert_equal :test,  @verb.test
-        assert_raise(Treat::Exception) { @verb.nonexistent }
-      end
-
-    end
-  end
-end

data/test/tc_extractors.rb

@@ -1,73 +0,0 @@
-# encoding: utf-8
-module Treat
-  module Tests
-    class TestExtractors < Test::Unit::TestCase
-
-      def setup
-        @time = Treat::Tests::English::Time
-        @date = Treat::Tests::English::Date
-        @doc = Treat::Tests::English::LongDoc
-        @word = Treat::Tests::English::Word
-        @col = Treat::Tests::English::Collection
-      end
-
-      def test_time
-        assert_nothing_raised { @time.time(:nickel) }
-      end
-      
-      def test_date
-        assert_equal 2011, @date.date(:chronic).year
-        assert_equal 2011, @date.date(:ruby).year
-      end
-
-      def test_topic_words
-        assert_nothing_raised { @col.topic_words(:lda) }
-      end
-      
-      def test_named_entity
-        p = 'Angela Merkel and Nicolas Sarkozy were the first ones to board the p'
-        assert_nothing_raised { @doc.named_entity(:stanford) }
-      end
-
-      def test_keywords
-        assert_nothing_raised do
-          topics = @col.topic_words(:lda)
-          @doc.keywords(:topics_frequency, :topic_words => topics)
-        end
-      end
-
-      def test_topics
-        assert_nothing_raised { @doc.topics(:reuters) }
-      end
-
-      def test_statistics
-        @doc.chunk.segment(:tactful).tokenize
-        assert_equal 1, @word.frequency_in(:document)
-        assert_nothing_raised { @word.tf_idf ; puts @word.tf_idf }
-        # assert_nothing_raised { @doc.statistics(:position_in) }
-        # assert_nothing_raised { @doc.statistics(:transition_matrix) }
-        # assert_nothing_raised { @doc.statistics(:transition_probability) }
-      end
-      
-      def test_language
-        assert_equal Treat.default_language, @doc.language
-        Treat.detect_language = true
-        assert_equal :eng, @doc.language
-        
-        a = 'I want to know God\'s thoughts; the rest are details. - Albert Einstein'
-        b = 'El mundo de hoy no tiene sentido, así que ¿por qué debería pintar cuadros que lo tuvieran? - Pablo Picasso'
-        c = 'Un bon Allemand ne peut souffrir les Français, mais il boit volontiers les vins de France. - Goethe'
-        d = 'Wir haben die Kunst, damit wir nicht an der Wahrheit zugrunde gehen. - Friedrich Nietzsche'
-
-        assert_equal :eng, a.language
-        assert_equal :spa, b.language
-        assert_equal :fre, c.language
-        assert_equal :ger, d.language
-        
-        # Reset defaults
-        Treat.detect_language = false
-      end
-      
-    end
-  end
-end