treat 0.1.4 → 0.2.0

data/lib/treat/viewable.rb

@@ -0,0 +1,29 @@
+module Treat
+  module Viewable
+    # Return the entity's string value in plain text format.
+    def to_string; @value; end
+    # An alias for #to_string.
+    def to_s; visualize(:txt); end
+    alias :to_str :to_s
+    # Return a shortened value of the entity's string value using [...].
+    def short_value(ml = 6); visualize(:short_value, :max_length => ml); end
+    # Return an informative string representation of the entity.
+    def inspect
+      s = "#{cl(self.class)} (#{@id.to_s})"
+      if caller_method(2) == :inspect
+        @id.to_s
+      else
+        dependencies = []
+        @dependencies.each do |dependency|
+          dependencies << "#{dependency.target}#{dependency.type}"
+        end
+        s += "  |  #{short_value.inspect}" +
+        "  |  #{@features.inspect}" +
+        "  | { #{dependencies.join(', ')} }"
+      end
+      s
+    end
+    # Print out an ASCII representation of the tree.
+    def print_tree; puts visualize(:tree); end
+  end
+end

data/lib/treat/visitable.rb

@@ -8,7 +8,7 @@ module Treat
         if group.type == :transformer
           if has_children?
             @children.each do |entity|
-              if group.has_target?(entity.class)
+              if group.has_target?(entity.class) && entity.id != id
                 entity.accept(group, klass, method, options)
               end
             end

data/test/tc_entity.rb

@@ -2,17 +2,14 @@ module Treat
   module Tests
     class TestEntity < Test::Unit::TestCase
       def setup
-        @text = Treat::Entities::Section.new
-        
+        @section = Treat::Entities::Section.new
         @sentence = Treat::Entities::Sentence.new
-        
-        @noun_phrase = Treat::Entities::Phrase.new
-        @noun_phrase.set :tag, 'NP'
-        @verb_phrase = Treat::Entities::Phrase.new
-        @verb_phrase.set :tag, 'VP'
-        @adj_phrase = Treat::Entities::Phrase.new
-        @adj_phrase.set :tag, 'ADJP'
-        
+        @noun_cons = Treat::Entities::Phrase.new
+        @noun_cons.set :tag, 'NP'
+        @verb_cons = Treat::Entities::Phrase.new
+        @verb_cons.set :tag, 'VP'
+        @adj_cons = Treat::Entities::Phrase.new
+        @adj_cons.set :tag, 'ADJP'
         @det = Treat::Entities::Word.new('The')
         @det.set :category, :determiner
         @det.set :tag, 'DT'
@@ -34,77 +31,87 @@ module Treat
         @verb.set :tag, 'VBG'
         @verb.set :tag_set, :penn
         @dot = Treat::Entities::Punctuation.new('.')
-        
-        @text << @sentence << [@noun_phrase, @verb_phrase, @dot]
-        @noun_phrase << [@det, @adj_phrase, @noun]
-        @adj_phrase << @adj
-        @verb_phrase << [@aux, @verb]
+        @section << @sentence << [@noun_cons, @verb_cons, @dot]
+        @noun_cons << [@det, @adj_cons, @noun]
+        @adj_cons << @adj
+        @verb_cons << [@aux, @verb]
       end
 
-      def test_respond_to_missing
-      
+      def test_viewable
+        s = 'Happiness is not an ideal of reason, but of imagination.'.tokenize
+        assert_nothing_raised do
+          # Return the string value of the sentence.
+          s.to_s
+          # Return a debug description of the sentence.
+          s.inspect
+          # Return a shortened version of the Sentence with [...]
+          s.short_value
+        end
       end
-      
+
       def test_registrable
-        assert_equal @text.token_registry, @verb.token_registry
-        assert_equal @noun, @text.token_registry[:id][@noun.id]
-        assert_equal [@noun], @text.token_registry[:value][@noun.value]  
+        assert_equal @section.token_registry, @verb.token_registry
+        assert_equal @noun, @section.token_registry[:id][@noun.id]
+        assert_equal [@noun], @section.token_registry[:value][@noun.value]
       end
 
-
       def test_delegatable_visitable
-        assert_raise(Treat::Exception) do 
-          @text.encoding(:nonexistent)
+        assert_raise(Treat::Exception) do
+          @section.encoding(:nonexistent)
         end
         assert_nothing_raised do
-          @text.language
+          @section.language
         end
       end
-      
+
       def test_type
-        assert_equal :section, @text.type
+        assert_equal :section, @section.type
       end
-      
+
       def test_printers
         assert_nothing_raised do
-          @text.to_s
-          @text.to_string
-          @text.short_value
-          @text.inspect
+          @section.to_s
+          @section.to_string
+          @section.short_value
+          @section.inspect
         end
       end
 
       def test_magic_methods
-        assert_equal @sentence, @text.sentence
-        assert_equal [@sentence], @text.sentences
-        assert_equal 1, @text.sentence_count
-        
-        assert_equal [@det], @text.words_with_value('The')
-        assert_equal [@verb], @text.words_with_tag('VBG')
-        
-        assert_equal @noun, @text.noun
-        assert_equal [@aux, @verb], @text.verbs
-        assert_equal 6, @text.token_count
-        
-        @text.each_sentence do |s|
+
+        assert_equal true, @sentence.is_sentence?
+        assert_equal true, @noun.is_noun?
+
+        assert_equal @sentence, @section.sentence
+        assert_equal [@sentence], @section.sentences
+        assert_equal 1, @section.sentence_count
+
+        assert_equal [@det], @section.words_with_value('The')
+        assert_equal [@verb], @section.words_with_tag('VBG')
+
+        assert_equal @noun, @section.noun
+        assert_equal [@aux, @verb], @section.verbs
+        assert_equal 6, @section.token_count
+
+        @section.each_sentence do |s|
           assert_equal @sentence, s
         end
-        @text.each_noun do |n|
+        @section.each_noun do |n|
           assert_equal @noun, n
         end
-        @text.each_with_value('The') do |x|
+        @section.each_with_value('The') do |x|
           assert_equal @det, x
         end
-        
+
         assert_equal @sentence, @noun.parent_sentence
       end
 
       def test_features
         @verb.set :test, :test
         assert_equal :test,  @verb.test
-        assert_raise(Treat::Exception) { @verb.nonexistent } 
+        assert_raise(Treat::Exception) { @verb.nonexistent }
       end
-      
+
     end
   end
 end

data/test/tc_extractors.rb

@@ -1,34 +1,38 @@
+# encoding: utf-8
 module Treat
   module Tests
     class TestExtractors < Test::Unit::TestCase
 
       def setup
-        @time = Treat::Tests::EnglishTime
-        @date = Treat::Tests::EnglishDate
-        @doc = Treat::Tests::EnglishLongDoc
-        @word = Treat::Tests::EnglishWord
+        @time = Treat::Tests::English::Time
+        @date = Treat::Tests::English::Date
+        @doc = Treat::Tests::English::LongDoc
+        @word = Treat::Tests::English::Word
+        @col = Treat::Tests::English::Collection
       end
 
       def test_time
-        assert_nothing_raised { @date.time(:chronic) }
-        assert_nothing_raised { @date.time(:native) }
-        assert_nothing_raised { @date.time(:nickel) }
+        assert_nothing_raised { @time.time(:nickel) }
+      end
+      
+      def test_date
+        assert_equal 2011, @date.date(:chronic).year
+        assert_equal 2011, @date.date(:ruby).year
       end
 
       def test_topic_words
-        assert_nothing_raised { @doc.topic_words(:lda) }
+        assert_nothing_raised { @col.topic_words(:lda) }
       end
-
-
+      
       def test_named_entity
-        # assert_nothing_raised { @doc.named_entity(:stanford) }
-        # assert_nothing_raised { @doc.named_entity(:abner) }
+        p = 'Angela Merkel and Nicolas Sarkozy were the first ones to board the p'
+        assert_nothing_raised { @doc.named_entity(:stanford) }
       end
 
       def test_keywords
-        assert_nothing_raised do 
-          topics = @doc.topic_words(:lda)
-          @doc.keywords(:topics_frequency, topic_words: topics)
+        assert_nothing_raised do
+          topics = @col.topic_words(:lda)
+          @doc.keywords(:topics_frequency, :topic_words => topics)
         end
       end
 
@@ -38,13 +42,32 @@ module Treat
 
       def test_statistics
         @doc.chunk.segment(:tactful).tokenize
-
-        assert_nothing_raised { @doc.statistics(:frequency_of, value: 'the') }
-        assert_nothing_raised { @word.statistics(:frequency_in) }
+        assert_equal 1, @word.frequency_in(:document)
+        assert_nothing_raised { @word.tf_idf ; puts @word.tf_idf }
         # assert_nothing_raised { @doc.statistics(:position_in) }
         # assert_nothing_raised { @doc.statistics(:transition_matrix) }
         # assert_nothing_raised { @doc.statistics(:transition_probability) }
       end
+      
+      def test_language
+        assert_equal Treat.default_language, @doc.language
+        Treat.detect_language = true
+        assert_equal :eng, @doc.language
+        
+        a = 'I want to know God\'s thoughts; the rest are details. - Albert Einstein'
+        b = 'El mundo de hoy no tiene sentido, así que ¿por qué debería pintar cuadros que lo tuvieran? - Pablo Picasso'
+        c = 'Un bon Allemand ne peut souffrir les Français, mais il boit volontiers les vins de France. - Goethe'
+        d = 'Wir haben die Kunst, damit wir nicht an der Wahrheit zugrunde gehen. - Friedrich Nietzsche'
+
+        assert_equal :eng, a.language
+        assert_equal :spa, b.language
+        assert_equal :fre, c.language
+        assert_equal :ger, d.language
+        
+        # Reset defaults
+        Treat.detect_language = false
+      end
+      
     end
   end
 end

data/test/tc_formatters.rb

@@ -3,25 +3,25 @@ module Treat
     class TestFormatters < Test::Unit::TestCase
 
       def setup
-        @doc = Treat::Tests::EnglishShortDoc
-        @html_doc = Treat::Tests::EnglishHtmlDoc
-        @sentence = Treat::Tests::EnglishSentence
+        @doc = Treat::Tests::English::ShortDoc
+        @sentence = Treat::Tests::English::Sentence
       end
 
       def test_readers
-        # How should we test this?
+        # This is done by loading a collection with all types of texts.
       end
 
-
       def test_serializers_and_unserializers
+        # Test roundtrip Ruby -> YAML -> Ruby -> YAML
         create_temp_file('yml') do |tmp|
-          @doc.serialize(:yaml).save(tmp)
+          @doc.serialize(:yaml, :file => tmp)
           doc = Treat::Entities::Document(tmp)
           assert_equal File.read(tmp).length, 
           doc.serialize(:yaml).length
         end
+        # Test roundtrip Ruby -> XML -> Ruby -> XML.
         create_temp_file('xml') do |tmp|
-          @doc.serialize(:xml).save(tmp)
+          @doc.serialize(:xml, :file => tmp)
           doc = Treat::Entities::Document(tmp)
           assert_equal File.read(tmp).length, 
           doc.serialize(:xml).length
@@ -32,7 +32,6 @@ module Treat
         assert_nothing_raised { @doc.visualize(:tree) }
         # assert_nothing_raised { @doc.visualize(:html) }
         assert_nothing_raised { @doc.visualize(:dot) }
-        assert_nothing_raised { @doc.visualize(:inspect) }
         assert_nothing_raised { @doc.visualize(:short_value) }
         assert_nothing_raised { @sentence.visualize(:standoff) }
       end

data/test/tc_inflectors.rb

@@ -2,38 +2,33 @@ module Treat
   module Tests
     class TestInflectors < Test::Unit::TestCase
 
-      def setup
-        @word = Treat::Tests::EnglishWord
-        @number = Treat::Tests::Number
-        @verb = Treat::Tests::EnglishVerb
-        @noun = Treat::Tests::EnglishNoun
-      end
-
       def test_lemmatizers
         # Not implemented yet.
       end
 
       def test_stemmers
-        assert_equal 'run', @word.stem(:porter)
-        assert_equal 'run', @word.stem(:porter_c)
-        assert_equal 'run', @word.stem(:uea)
+        assert_equal 'run', 'running'.stem(:porter)
+        assert_equal 'run', 'running'.stem(:porter_c)
+        assert_equal 'run', 'running'.stem(:uea)
       end
-    end
 
-    def test_conjugators
-      assert_equal 'running', @verb.present_participle
-      assert_equal 'run', @verb.infinitive
-      assert_equal 'run', @verb.plural
-    end
+      def test_conjugators
+        assert_equal 'run', 'running'.infinitive
+        assert_equal 'running', 'run'.present_participle
+        assert_equal 'run', 'runs'.plural_verb
+      end
 
-    def test_declensors
-      assert_equal 'geese', @noun.plural
-    end
+      def test_declensors
+        assert_equal 'inflections', 'inflection'.plural(:linguistics)
+        assert_equal 'inflections', 'inflection'.plural(:english)
+        assert_equal 'inflection', 'inflections'.singular(:english)
+      end
 
-    def test_ordinal_and_cardinal_words
-      assert_equal 'twenty', @number.cardinal_words
-      assert_equal 'twentieth', @number.ordinal_words
-    end
+      def test_ordinal_and_cardinal_words
+        assert_equal 'twenty', 20.cardinal_words
+        assert_equal 'twentieth', 20.ordinal_words
+      end
 
+    end
   end
-end
+end

data/test/tc_lexicalizers.rb

@@ -2,35 +2,28 @@ module Treat
   module Tests
     class TestLexicalizers < Test::Unit::TestCase
       
-      def setup
-        @word = Treat::Tests::EnglishWord
-        @sentence = Treat::Tests::EnglishSentence.parse
-      end
-      
       def test_category
-        assert_equal :verb, @word.category(:from_tag)
+        assert_equal :verb, 'visualize'.category(:from_tag, :tagger => :stanford)
+        assert_equal :noun, 'inflection'.category(:from_tag, :tagger => :brill)
+        assert_equal :adjective, 'sweet'.category(:from_tag, :tagger => :lingua)
       end
       
       def test_synsets
-        # assert_nothing_raised { @word.synsets(:rita_wn) }
-        assert_nothing_raised { @word.synsets(:wordnet) }
-        assert_nothing_raised { @word.synonyms(:wordnet) }
-        assert_nothing_raised { @word.antonyms(:wordnet) }
-        assert_nothing_raised { @word.hyponyms(:wordnet) }
-        assert_nothing_raised { @word.hypernyms(:wordnet) }
+        assert_equal 'mature', 'ripe'.synonyms(:wordnet)[0]
+        # assert_equal 'green', ' ripe'.antonyms(:wordnet)[0]
+        assert_equal 'beverage', 'coffee'.hypernyms(:wordnet)[0]
+        assert_equal 'gravy', 'juice'.hyponyms(:wordnet)[0]
       end
       
       def test_linkages
-        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :main_verb) }
-        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :subject) }
-        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :object) }
-        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :patient) }
+        sentence = 'Good is bad, but bad is not good'
+       # assert_equal sentence.parse(:enju).linkages
       end
       
       def test_taggers
-        assert_nothing_raised { @word.tag(:brill) }
-        assert_nothing_raised { @word.tag(:lingua) }
-        assert_nothing_raised { @word.tag(:stanford) }
+        assert_equal 'VBG', 'running'.tag(:stanford)
+        assert_equal 'VBG', 'running'.tag(:brill)
+        assert_equal 'VBG', 'running'.tag(:lingua)
       end
       
     end

data/test/tc_processors.rb

@@ -3,31 +3,45 @@ module Treat
     class TestProcessors < Test::Unit::TestCase
       
       def setup
-        @doc = Treat::Tests::EnglishShortDoc 
+        @doc = Treat::Tests::English::ShortDoc 
       end
       
       def test_tokenizers
-        assert_nothing_raised { @doc.tokenize(:macintyre) }
-        assert_nothing_raised { @doc.tokenize(:multilingual) }
-        assert_nothing_raised { @doc.tokenize(:perl) }
-        assert_nothing_raised { @doc.tokenize(:punkt) }
-        assert_nothing_raised { @doc.tokenize(:stanford) }
-        assert_nothing_raised { @doc.tokenize(:tactful) }
+        words = ['A', 'sentence', 'to', 'tokenize']
+        tokenize_map = lambda do |worker, o={}|
+          'A sentence to tokenize'.
+          tokenize(worker, o).words.map { |w| w.value }
+        end
+        assert_equal words, tokenize_map.call(:macintyre)
+        assert_equal words, tokenize_map.call(:multilingual)
+        assert_equal words, tokenize_map.call(:perl)
+        assert_equal words, tokenize_map.call(:punkt)
+        assert_equal words, tokenize_map.call(:stanford, :silence => true)
+        assert_equal words, tokenize_map.call(:tactful)
       end
       
       def test_segmenters
-        assert_nothing_raised { @doc.segment(:punkt) }
-        assert_nothing_raised { @doc.segment(:stanford) }
-        assert_nothing_raised { @doc.segment(:tactful) }
+        sentences = ['This is sentence 1.', 'This is sentence 2.']
+        segment_map = lambda do |worker,o={}| 
+          'This is sentence 1. This is sentence 2.'.
+          segment(worker, o).sentences.map { |s| s.value }
+        end
+        assert_equal sentences, segment_map.call(:punkt)
+        assert_equal sentences, segment_map.call(:stanford, :silence => true)
+        assert_equal sentences, segment_map.call(:tactful)
       end
   
       def test_chunkers
-        assert_nothing_raised { @doc.chunk(:txt) }
+        title = 'This is a title!'
+        paragraph = 'This is sentence 1. This is a potential sentence inside a pargraph describing the wonders of the world.'
+        s = "This is a title!\nThis is sentence 1. This is a potential sentence inside a pargraph describing the wonders of the world.".chunk
+        assert_equal title, s.title.value
+        assert_equal paragraph, s.paragraph.value
       end
           
       def test_parsers
         assert_nothing_raised { @doc.segment.parse(:enju) }
-        assert_nothing_raised { @doc.segment.parse(:stanford) }
+        assert_nothing_raised { @doc.segment.parse(:stanford, :silence => true) }
       end
       
     end