rbbt-text 0.6.3 → 1.0.0

data/test/rbbt/corpus/test_corpus.rb

@@ -1,99 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
-require 'rbbt/corpus/corpus'
-require 'rbbt/corpus/sources/pubmed'
-
-class Document
-  define :sentences do 
-    require 'rbbt/nlp/nlp'
-    NLP.geniass_sentence_splitter(text)
-  end
-
-  define :genes do 
-    require 'rbbt/ner/abner'
-    Abner.new.entities(text)
-  end
-end
-
- 
-class TestCorpus < Test::Unit::TestCase
-
-  def test_add_document
-    pmid = "19458159"
-
-    text = PubMed.get_article(pmid).text
-
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-
-    assert corpus.find(:pubmed, pmid).empty?
-
-    corpus.add_document(text, :pubmed, pmid, :abstract)
-
-    assert corpus.find(:pubmed, pmid).any?
-    assert corpus.find(:pubmed, pmid, :fulltext).empty?
-    assert corpus.find(:pubmed, pmid, :abstract).any?
-
-    assert corpus.find(:pubmed, pmid).first.text =~ /SENT/
-  end
-
-  def test_add_pmid
-    pmid = "19465387"
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-    corpus.add_pmid(pmid, :abstract)
-
-    assert corpus.exists? :pubmed, pmid
-    assert corpus.exists? :pubmed, pmid, :abstract
-    assert_equal false, corpus.exists?(:pubmed, pmid, :fulltext) 
-  end
-
-  def test_find_all
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-    corpus.add_pmid("19458159", :abstract)
-    corpus.add_pmid("19465387", :abstract)
-
-    all = corpus.find
-
-    assert_equal 2, all.length
-    assert all.select{|document| document.id == "19458159"}.any?
-    assert all.select{|document| document.id == "19465387"}.any?
-  end
-
-  def test_doc_sentences
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-    corpus.add_pmid("19458159", :abstract)
-
-    sentences = corpus.find.collect{|doc| doc.sentences}.flatten
-
-    assert corpus.find.first.sentences.length > 0 
-    assert corpus.find.first.sentences.sort_by{|s| s.offset}.first =~ /Semantic features in Text/i
-
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-    assert corpus.find.first.sentences.sort_by{|s| s.offset}.first =~ /Semantic features in Text/i
-  end
-
-  def test_doc_genes
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-    corpus.add_pmid("21611789", :abstract)
-    
-    assert corpus.find(:pubmed, "21611789").first.genes.include? "CDKAL1"
-  end
-
-  def test_genes
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-    corpus.add_pmid("21611789", :abstract)
-
-    assert corpus.find.collect{|d| d.genes}.flatten.include? "CDKAL1"
-  end
-
-  def test_index
-    corpus = Corpus.new(Rbbt.tmp.test.Corpus.find :user)
-    corpus.add_pmid("21611789", :abstract)
-
-    document = corpus.find(:pubmed, "21611789").first
-
-    genes = corpus.find.collect{|d| d.genes}.flatten.select{|gene| gene == "CDKAL1"}
-
-    assert genes.collect{|gene|
-      document.sentences_at(gene.offset)
-    }.flatten.length >  1
-  end
-end

data/test/rbbt/corpus/test_document.rb

@@ -1,236 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
-require 'rbbt/corpus/document'
-require 'test/unit'
-
-module TokenEntity
-  extend Annotation
-  include Segment
-  self.annotation :original
-end
-class Document
-
-  def tokenize(text)
-    Token.tokenize(text).collect do |token|
-      TokenEntity.setup(token.dup, token.offset, token.original)
-    end
-  end
-
-  define :sentences do 
-    require 'rbbt/nlp/nlp'
-    NLP.geniass_sentence_splitter(text)
-  end
-
-  define :tokens do
-    require 'rbbt/ner/segment/token'
-    tokenize(text)
-  end
- 
-  define :long_words do
-    require 'rbbt/ner/segment/token'
-    tokenize(text).select{|tok| tok.length > 5}
-  end
-
-  define :short_words do
-    require 'rbbt/ner/segment/token'
-    tokenize(text).select{|tok| tok.length < 5}
-  end
- 
-  define :even_words do
-    require 'rbbt/ner/segment/token'
-    tokenize(text).select{|tok| tok.length % 2 == 0}
-  end
-
-  define :missing do
-    []
-  end
-
-  define :tokens_again do
-    raise "This should be here already"
-  end
-end
-
-class TestDocument < Test::Unit::TestCase
-
-  def setup
-    global_fields = ["Start", "End", "JSON", "Document ID", "Entity Type"]
-    $persistence = TSV.setup({})
-    $tchash_persistence = Persist.open_tokyocabinet(Rbbt.tmp.test.document.persistence.find(:user), true, :tsv)
-    $global_persistence = TSV.setup({}, :key => "ID", :fields => global_fields)
-    $tchash_global_persistence = TSV.setup(Persist.open_tokyocabinet(Rbbt.tmp.test.global.persistence.find(:user), true, :list), :key => "ID", :fields => global_fields + ["Document ID", "Entity Type"])
-    $tchash_global_persistence.read
-    $tchash_global_persistence.write
-
-    Document.class_eval do
-
-      persist :sentences
-      persist_in_tsv :tokens, :literal
-      persist_in_tsv :long_words, $tchash_persistence, :literal
-      persist_in_global_tsv :short_words, $global_persistence
-      persist_in_global_tsv :even_words, $tchash_global_persistence
-      persist_in_global_tsv :missing, $tchash_global_persistence
-    end
-  end
-
-  def test_annotations
-
-    text =<<-EOF
-This is a 
-sentence. This is
-another sentence.
-    EOF
-
-    doc = Document.new
-    doc.text = text
-
-    assert_equal 2, doc.sentences.length 
-    assert_equal 10, doc.tokens.length 
-  end
-
-  def test_annotation_load
-    text =<<-EOF
-This is a 
-sentence. This is
-another sentence.
-    EOF
-
-    doc = Document.new
-    doc.text = text * 10
-
-    sentence = doc.sentences.last
-    doc.load_into sentence, :tokens 
-    assert_equal 5, sentence.tokens.length
-    assert_equal "another", sentence.tokens[2]
-    assert_equal sentence.offset + 0, sentence.tokens[0].offset
-  end
-
-  def test_annotation_persistence
-    text =<<-EOF
-This is a 
-sentence. This is
-another sentence. 
-    EOF
-
-    text *= 10
-
-    TmpFile.with_file do |dir|
-      FileUtils.mkdir_p dir
-
-      doc = Document.new(dir)
-      doc.text = text
-      doc.sentences
-
-      doc = Document.new(dir)
-      doc.text = text
-
-      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
-      doc.load_into sentence, :tokens 
-
-      assert_equal 5, sentence.tokens.length
-      assert_equal "another", sentence.tokens[2]
-      assert_equal sentence.offset + 0, sentence.tokens[0].offset
-    end
-  end
-
-  def test_range_persistence
-    text =<<-EOF
-This is a 
-sentence. This is
-another sentence. 
-    EOF
-
-    text *= 10
-
-    TmpFile.with_file do |dir|
-      FileUtils.mkdir_p dir
-
-      doc = Document.new(dir)
-      doc.text = text
-
-      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
-      Misc.benchmark(1) do
-        doc = Document.new(dir)
-        doc.text = text
-
-        doc.load_into sentence, :tokens, :persist => true
-        assert_equal 5, sentence.tokens.length
-        assert_equal "another", sentence.tokens[2]
-        assert_equal sentence.offset + 0, sentence.tokens[0].offset
-        assert_equal sentence.offset + 5, sentence.tokens[1].offset
-      end
-    end
-  end
-
-  def test_annotation_persistence_in_tsv
-    text =<<-EOF
-This is a 
-sentence. This is
-another sentence. 
-    EOF
-
-    TmpFile.with_file do |dir|
-      FileUtils.mkdir_p dir
-
-
-      doc = Document.new(dir)
-      doc.text = text * 10
-      doc.sentences
-
-      doc = Document.new(dir)
-      doc.text = text * 10
-
-      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
-
-      doc.load_into sentence, :tokens, :long_words
-
-      assert_equal 5, sentence.tokens.length
-      assert_equal "another", sentence.tokens[2]
-      assert_equal sentence.offset + 0, sentence.tokens[0].offset
-
-      assert_equal 2, sentence.long_words.length
-      doc = Document.new(dir)
-      doc.text = text * 10
-      doc.sentences
-      assert_equal sentence, doc.sentences.sort_by{|sentence| sentence.offset}.last
-
-      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
-      doc.load_into sentence, :tokens, :long_words
-
-      assert_equal 2, sentence.long_words.length
-      assert_equal %w(another sentence), sentence.long_words
-      assert_equal sentence.offset + "This is ".length, sentence.long_words[0].offset
-    end
-  end
-
-  def test_annotation_persistence_in_global
-    text =<<-EOF
-This is a 
-sentence. This is
-another sentence. 
-    EOF
-
-    TmpFile.with_file do |dir|
-      FileUtils.mkdir_p dir
-
-
-      global_persistence = TSV.setup({}, :fields => %w(Start End annotation_types JSON) + ["Document ID", "Entity Type"])
-      doc = Document.new(dir, nil, nil, global_persistence)
-      doc.text = text * 10
-      doc.docid = "TEST"
-
-      doc.sentences
-
-      doc = Document.new(dir)
-      doc.text = text * 10
-      doc.docid = "TEST"
-
-      sentence = doc.sentences.sort_by{|sentence| sentence.offset}.last
-
-      doc.load_into sentence, :tokens, :long_words, :short_words, :even_words
-
-      assert_equal 3, sentence.short_words.length
-      assert_equal 3, sentence.even_words.length
-    end
-  end
-end
-
-