treat 0.2.5 → 1.0.0

data/spec/document.rb

@@ -0,0 +1,93 @@
+require_relative '../lib/treat'
+
+describe Treat::Entities::Document do
+
+  describe "Extractable" do
+
+    describe "#topics" do
+
+      it "returns a list of general topics the document belongs to" do
+        #doc = Treat::Entities::Document.new(
+        #Treat.spec + 'samples/mathematicians/archimedes.abw').read(:abw)
+        #doc.do(:chunk, :segment, :tokenize)
+        #puts doc.topics.inspect
+      end
+
+    end
+
+  end
+
+  describe "Buildable" do
+
+    describe "#build" do
+
+      context "when supplied with a readable file name" do
+        it "opens the file and reads its " +
+        "content into a document" do
+          f = Treat.spec + 'samples/mathematicians/leibniz.txt'
+          d = Treat::Entities::Document.build(f)
+          d.should be_an_instance_of Treat::Entities::Document
+          d.to_s.index('Gottfried Leibniz').should_not eql nil
+        end
+      end
+
+      context "when supplied with a url" do
+        it "downloads the file the URL points to and opens " +
+        "a document with the contents of the file" do
+          url = 'http://www.rubyinside.com/nethttp-cheat-sheet-2940.html'
+          d = Treat::Entities::Document.build(url)
+          d.should be_an_instance_of Treat::Entities::Document
+          d.to_s.index('Rubyist').should_not eql nil
+        end
+      end
+
+      context "when called with anything else than a " +
+      "readable file name or url" do
+
+        it "raises an exception" do
+          lambda do
+            Treat::Entities::Document.build('nonexistent')
+          end.should raise_error
+        end
+
+      end
+
+    end
+
+  end
+
+  describe "Processable" do
+
+    describe "#chunk" do
+
+      context "when called on an HTML document" do
+        doc = Treat::Entities::Document.new(
+        Treat.spec + 'samples/mathematicians/euler.html').read(:html)
+        it "splits the HTML document into sections, " +
+        "titles, paragraphs and lists" do
+          doc.chunk
+          doc.title_count.should eql 1
+          doc.title.to_s.should eql "Leonhard Euler (1707-1783)"
+          doc.paragraph_count.should eql 5
+        end
+
+      end
+
+      context "when called on a text document" do
+
+        doc = Treat::Entities::Document.new(Treat.spec +
+        'samples/mathematicians/leibniz.txt').read(:txt)
+        it "splits the document into titles and paragraphs" do
+          doc.chunk
+          doc.title_count.should eql 1
+          doc.title.to_s.should eql "Gottfried Leibniz (1646-1716)"
+          doc.paragraph_count.should eql 6
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/spec/entity.rb

@@ -0,0 +1,408 @@
+require_relative '../lib/treat'
+
+describe Treat::Entities::Entity do
+
+  before do
+
+    @paragraph = Treat::Entities::Paragraph.new
+    @sentence = Treat::Entities::Sentence.new
+    @noun_phrase = Treat::Entities::Phrase.new
+    @noun_phrase.set :tag, 'NP'
+    @verb_phrase = Treat::Entities::Phrase.new
+    @verb_phrase.set :tag, 'VP'
+    @adj_phrase = Treat::Entities::Phrase.new
+    @adj_phrase.set :tag, 'ADJP'
+    @det = Treat::Entities::Word.new('The')
+    @det.set :category, :determiner
+    @det.set :tag, 'DT'
+    @adj = Treat::Entities::Word.new('lazy')
+    @adj.set :category, :adjective
+    @adj.set :tag, 'JJ'
+    @noun = Treat::Entities::Word.new('fox')
+    @noun.set :category, :noun
+    @noun.set :tag, 'NN'
+    @aux = Treat::Entities::Word.new('is')
+    @aux.set :category, :verb
+    @aux.set :tag, 'VBZ'
+    @verb = Treat::Entities::Word.new('running')
+    @verb.set :category, :verb
+    @verb.set :tag, 'VBG'
+    @dot = Treat::Entities::Punctuation.new('.')
+    @dot.set :tag, '.'
+    @paragraph << @sentence << [@noun_phrase, @verb_phrase, @dot]
+    @noun_phrase << [@det, @adj_phrase, @noun]
+    @adj_phrase << @adj
+    @verb_phrase << [@aux, @verb]
+
+  end
+
+
+  describe "Checkable" do
+
+    describe "#check_has(feature, do_it = true) " do
+      
+      it "checks if an entity has the feature; if not, " +
+      "calls the default worker to get the feature if do_it " +
+      "is set to true; if the entity doesn't have the feature " +
+      " and do_it is set to false, it raises an exception." do
+        
+        # NOT PASSING! Dependence on caller method.
+
+       # lambda {  '$'.to_entity.check_has(:tag, false) }.should raise_error Treat::Exception
+ 
+      end
+
+    end
+
+  end
+
+  describe "Countable" do
+
+    describe "#position" do
+
+      it "returns the position of the entity in its parent, sarting at 1" do
+        @noun_phrase.position.should eql 1
+        @det.position.should eql 1
+      end
+
+    end
+
+=begin
+    
+    describe "#frequency" do
+
+      it "returns the frequency of the entity's value in the root" do
+         @det.frequency.should eql 1
+      end
+
+    end
+
+
+    describe "#frequency_in(parent_type = nil)" do
+
+      it "returns the position of the entity's value "+
+         "in the supplied parent type, or root if nil" do
+           @noun_phrase.frequency_in(:sentence).should eql 1
+          
+      end
+      
+    end
+
+=end
+
+  end
+
+  describe "Delegatable" do
+
+    describe "#self.call_worker" do
+
+      it "finds the worker class to " +
+      "perform a task and delegates the task to it " do
+
+        Treat::Entities::Entity.call_worker(
+        '$'.to_entity, :tag, :lingua,
+        Treat::Lexicalizers::Taggers, {}).should
+        eql @sentence.tag(:lingua)
+
+      end
+
+    end
+
+  end
+
+  describe "Exportable" do
+
+    context "when supplied with a classification to export" do
+      classification = Treat::Classification.new(:word, :tag, :is_keyword?)
+      it "returns a data set with the exported features" do
+        ds = @sentence.export(classification)
+        ds.classification.should eql classification
+        ds.labels.should eql [:tag]
+        ds.ids.should eql @sentence.words.map { |w| w.id }
+        ds.items.should eql [
+          ["DT", false], ["JJ", false],
+          ["NN", false], ["VBZ", false],
+          ["VBG", false]
+        ]
+      end
+    end
+
+  end
+
+  describe "Iterable" do
+
+    describe "#each { |child| ... }" do
+      it "yields each direct child of a node" do
+        a = []
+        @sentence.each do |child|
+          a << child
+        end
+        a.should eql [@noun_phrase, @verb_phrase, @dot]
+      end
+    end
+
+    describe "#each_entity(*entity_types) { |entity| ... }" do
+
+      context "when called with no arguments" do
+        it "recursively yields each element in " +
+        "the tree, including itself, top-down " +
+        "first then left to right" do
+
+          a = []
+          @sentence.each_entity do |e|
+            a << e
+          end
+
+          a.should eql [@sentence, @noun_phrase, @det,
+            @adj_phrase, @adj, @noun,
+          @verb_phrase, @aux, @verb, @dot]
+
+        end
+      end
+
+      context "when called with one or more entity " +
+      "types supplied as lowercase symbols" do
+        it "recursively yields all elements with the given type(s), "+
+        "including the receiver if it matches on of the types" do
+          a = []
+          @sentence.each_entity(:phrase, :punctuation) do |e|
+            a << e
+          end
+          a.should eql [@sentence, @noun_phrase,
+          @adj_phrase, @verb_phrase, @dot]
+        end
+      end
+
+    end
+  end
+
+  describe "Magical" do
+
+    describe "#<entity or word type> - e.g. " +
+    "#title, #paragraph, etc. and #adjective, #noun, etc." do
+
+      it "return the first entity with the corresponding " +
+      "type inside another entity, but raises an exception "+
+      "the type occurs more than once in the entity" do
+        @paragraph.sentence.should eql @sentence
+      end
+
+    end
+
+
+    describe "#<entity or word type>s - e.g. " +
+    "#sections, #words, etc. and #nouns, #adverbs, etc." do
+
+      it "return an array of the entities with the " +
+      "corresponding type in the subtree of an entity" do
+        @paragraph.phrases.should eql [@sentence,
+        @noun_phrase, @adj_phrase, @verb_phrase]
+      end
+
+    end
+
+    describe "#each_<entity type> - e.g. " +
+    "#each_sentence, #each_word, etc." do
+
+      it "yields each of the entities with the " +
+      "corresponding type in the subtree of an entity" do
+        a = []
+
+        @paragraph.each_phrase { |p| a << p }
+        a.should eql [@sentence, @noun_phrase,
+        @adj_phrase, @verb_phrase]
+
+      end
+
+    end
+
+    describe "#<entity or word type>_count - e.g. " +
+    "#sentence_count, #paragraph_count, etc. and " +
+    "#noun_count, #verb_count, etc." do
+
+      it "return the number of entities with the " +
+      "corresponding type inside another entity" do
+        @paragraph.sentence_count.should eql 1
+        @paragraph.phrase_count.should eql 4
+      end
+
+    end
+
+    describe "#<entity or word type>_with_<feature>(value) - " +
+    "e.g. #word_with_id(x) or #adverb_with_value('seemingly')" do
+
+      it "return the entity with the corresponding type " +
+      "that have [feature] set to the supplied value; raise" +
+      "a warning if there are many entities of that type" do
+        @paragraph.word_with_value('The').should eql @det
+        @paragraph.token_with_tag('.').should eql @dot
+        @sentence.phrase_with_tag('NP').should eql @noun_phrase
+      end
+
+    end
+
+    describe "#<entity or word type>s_with_<feature>(value) - " +
+    "e.g. #phrases_with_tag('NP'), #nouns_with_value('foo')" do
+
+      it "return an array of the entities with the " +
+      "corresponding type that have [feature] set to "+
+      "the supplied value" do
+        @paragraph.words_with_value('The').should eql [@det]
+        @paragraph.tokens_with_tag('.').should eql [@dot]
+        @sentence.phrases_with_tag('NP').should eql [@noun_phrase]
+      end
+
+    end
+
+    describe "#parent_<entity type> - e.g. " +
+    "#parent_document, #parent_collection, etc." do
+
+      it "return the first ancestor of the entity " +
+      "that has the supplied type, or nil if none" do
+        @sentence.parent_paragraph.should eql @paragraph
+        @adj.parent_sentence.should eql @sentence
+      end
+
+    end
+
+    describe "#frequency_in_<entity type> - e.g. " +
+    "#frequency_in_collection, #frequency_in_document, etc." do
+
+      it "return the frequency of this entity's value " +
+      "in the parent entity with the corresponding type" do
+        @adj.frequency_in_sentence.should eql 1
+      end
+
+    end
+
+  end
+
+  describe "Stringable" do
+
+    describe "#to_string" do
+      it "returns the true text value of the entity " +
+      "or an empty string if it has none" do
+        @paragraph.to_string.should eql ''
+        @noun.to_string.should eql 'fox'
+      end
+    end
+
+    describe "#to_s" do
+      it "returns the string value of the " +
+      "entity or its full subtree" do
+        @paragraph.to_s.should
+        eql 'The lazy fox is running.'
+        @noun.to_s.should eql 'fox'
+      end
+    end
+
+    describe "#inspect" do
+      it "returns an informative string " +
+      "concerning the entity" do
+        @paragraph.inspect.should
+        be_an_instance_of String
+      end
+    end
+
+    describe "#short_value" do
+      it "returns a shortened version of the " +
+      "entity's string value" do
+        @paragraph.short_value.should
+        eql 'The lazy fox is running.'
+      end
+    end
+
+  end
+  
+  describe "Formatters" do 
+    
+    describe "#serialize" do
+      
+      before :all do 
+        @serializers = [:xml, :yaml] # Treat::Languages::All::Serializers
+        @txt = "The story of the fox. The quick brown fox jumped over the lazy dog."
+      end
+      
+      context "when called with a file to save to" do
+        
+        it "serializes a document to the supplied format" do
+          
+          @serializers.each do |ser|
+            f = Treat.spec + 'test.' + ser.to_s
+            s = Treat::Entities::Paragraph.new(@txt)
+            s.do(:segment, :tokenize)
+            s.serialize(ser, :file => f)
+            d = Treat::Entities::Document.build(f)
+            d.to_s.should eql @txt
+            d.size.should eql s.size
+            d.token_count.should eql s.token_count
+            d.tokens[0].id.should eql s.tokens[0].id
+            File.delete(f)
+          end
+          
+        end
+        
+      end
+      
+    end
+    
+    describe "#unserialize" do
+      
+    end
+    
+  end
+
+  describe "Extractors" do
+
+    describe "#language" do
+      context "when language detection is disabled " +
+      "(Treat.detect_language is set to false)" do
+        it "returns the default language (Treat.default_language)" do
+          Treat.detect_language = false
+          Treat.default_language = :test
+          s = 'Les grands hommes ne sont pas toujours grands, dit un jour Napoleon.'
+          s.language.should eql :test
+          Treat.default_language = :eng
+        end
+      end
+
+      context "when language detection is enabled " +
+      "(Treat.detect_language is set to true)" do
+
+        it "guesses the language of the entity" do
+
+          Treat.detect_language = true
+          a = 'I want to know God\'s thoughts; the rest are details. - Albert Einstein'
+          b = 'El mundo de hoy no tiene sentido, asi que por que deberia pintar cuadros que lo tuvieran? - Pablo Picasso'
+          c = 'Un bon Allemand ne peut souffrir les Francais, mais il boit volontiers les vins de France. - Goethe'
+          d = 'Wir haben die Kunst, damit wir nicht an der Wahrheit zugrunde gehen. - Friedrich Nietzsche'
+          a.language.should eql :eng
+          b.language.should eql :spa
+          c.language.should eql :fre
+          d.language.should eql :ger
+
+          # Reset default
+          Treat.detect_language = false
+        end
+
+      end
+
+    end
+
+  end
+
+end
+
+
+=begin
+
+
+def test_visualizers
+  assert_nothing_raised { @doc.visualize(:tree) }
+  # assert_nothing_raised { @doc.visualize(:html) }
+  assert_nothing_raised { @doc.visualize(:dot) }
+  assert_nothing_raised { @doc.visualize(:short_value) }
+  assert_nothing_raised { @sentence.visualize(:standoff) }
+end
+
+=end