rbbt-text 1.1.9 → 1.3.3

data/test/rbbt/test_document.rb

@@ -0,0 +1,14 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'test_helper.rb')
+require 'rbbt/document'
+
+class TestDocument < Test::Unit::TestCase
+
+  def test_docid
+    text = "This is a document"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    assert_equal ["TEST", "test_doc1", nil, Misc.digest(text)] * ":", text.docid
+  end
+
+end
+

data/test/rbbt/test_segment.rb

@@ -0,0 +1,182 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'test_helper.rb')
+require 'rbbt/document'
+require 'rbbt/document/corpus'
+require 'rbbt/segment'
+
+class TestSegment < Test::Unit::TestCase
+  def test_segment
+    text = "This is a document"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    segment = Segment.setup("is", :offset => text.index("is"), :docid => text.docid)
+
+    assert_equal text.docid + ":" + segment.offset.to_s + ".." + segment.eend.to_s,  segment.segid
+  end
+
+  def test_segid
+    text = "This is a document"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = Document::Corpus.setup({})
+
+    corpus.add_document(text)
+
+    segment = Segment.setup("is", :offset => text.index("is"), :docid => text.docid)
+
+    segid = segment.segid(corpus)
+
+    segment = segid.segment
+    assert_equal "is", segment
+  end
+
+  def test_info
+    segment = "test"
+    segment.extend Segment
+    segment.offset = 10
+    assert segment.info.include? :offset
+  end
+
+  def test_sort
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = Document::Corpus.setup({})
+
+    corpus.add_document(text)
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = text.index gene1
+    gene1.docid = text.docid
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = text.index gene2
+    gene2.docid = text.docid
+
+    assert_equal [gene1,gene2], Segment.sort([gene2,gene1])
+
+    assert_equal [gene1,gene2], Segment.sort([gene2.segid(corpus),gene1.segid(corpus)]).collect{|segid| segid.segment}
+  end
+
+  def test_clean_sort
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = Document::Corpus.setup({})
+
+    corpus.add_document(text)
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = text.index gene1
+    gene1.docid = text.docid
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = text.index gene2
+    gene2.docid = text.docid
+
+    gene3 = "TP53 gene"
+    gene3.extend Segment
+    gene3.offset = text.index gene1
+    gene3.docid = text.docid
+
+    assert_equal [gene1,gene2], Segment.sort([gene2,gene1])
+
+    assert_equal [gene3,gene2], Segment.clean_sort([gene2,gene1,gene3])
+  end
+
+  def test_split
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = Document::Corpus.setup({})
+
+    corpus.add_document(text)
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = text.index gene1
+    gene1.docid = text.docid
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = text.index gene2
+    gene2.docid = text.docid
+
+    gene3 = "TP53 gene"
+    gene3.extend Segment
+    gene3.offset = text.index gene1
+    gene3.docid = text.docid
+
+    assert_equal ["This sentence mentions the ", gene3, " and the ", gene2, " protein"], Segment.split(text, [gene2, gene1, gene3])
+
+    assert_equal ["This sentence mentions the ", gene3, " and the ", gene2, " protein"], Segment.split(text, [gene2, gene1, gene3].collect{|s| s.segid})
+  end
+
+
+  def test_align
+    text =<<-EOF
+Atypical teratoid/rhabdoid tumors (AT/RTs) are highly aggressive brain tumors of early childhood poorly responding to therapy.
+    EOF
+
+    parts = text.split(/\W/)
+    Segment.align(text, parts)
+
+    assert_equal "Atypical teratoid/".length, parts.select{|s| s == "rhabdoid"}.first.offset
+
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    parts = text.split(/\W/)
+    Segment.align(text, parts)
+
+    assert_equal parts.first.docid, text.docid
+  end
+
+  def test_segment_index
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = Document::Corpus.setup({})
+
+    corpus.add_document(text)
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = text.index gene1
+    gene1.docid = text.docid
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = text.index gene2
+    gene2.docid = text.docid
+
+    gene3 = "TP53 gene"
+    gene3.extend Segment
+    gene3.offset = text.index gene1
+    gene3.docid = text.docid
+
+    index = Segment.index([gene1, gene2, gene3], corpus)
+    assert_equal "CDK5R1", index[gene2.offset + 1].segment.first
+
+    TmpFile.with_file do |fwt|
+      index = Segment.index([gene1, gene2, gene3], corpus, fwt)
+      assert_equal %w(CDK5R1), index[gene2.offset + 1].segment
+      index = Segment.index([gene1, gene2, gene3], corpus, fwt)
+      assert_equal %w(CDK5R1), index[gene2.offset + 1].segment
+    end
+
+    index = Segment.index([gene1, gene2, gene3].collect{|s| s.segid}, corpus)
+    assert_equal "CDK5R1", index[gene2.offset + 1].segment.first
+
+    TmpFile.with_file do |fwt|
+      index = Segment.index([gene1, gene2, gene3].collect{|s| s.segid}, corpus, fwt)
+      assert_equal %w(CDK5R1), index[gene2.offset + 1].segment
+      index = Segment.index([gene1, gene2, gene3].collect{|s| s.segid}, corpus, fwt)
+      assert_equal %w(CDK5R1), index[gene2.offset + 1].segment
+    end
+  end
+
+end
+

data/test/test_helper.rb

@@ -6,7 +6,7 @@ require 'rbbt'
 require 'rbbt/persist'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/log'
-require 'rbbt/text/corpus'
+#require 'rbbt/text/corpus'
 
 class Test::Unit::TestCase
   def get_test_datafile(file)
@@ -22,8 +22,10 @@ class Test::Unit::TestCase
     FileUtils.rm_rf Rbbt.tmp.test.find :user
     Persist::CONNECTIONS.values.each do |c| c.close end
     Persist::CONNECTIONS.clear
-    Corpus::DocumentRepo::TC_CONNECTIONS.values.each do |c| c.close end
-    Corpus::DocumentRepo::TC_CONNECTIONS.clear
+    if defined? Corpus
+      Corpus::DocumentRepo::TC_CONNECTIONS.values.each do |c| c.close end
+      Corpus::DocumentRepo::TC_CONNECTIONS.clear
+    end
   end
 
 end

data/test/test_spaCy.rb

@@ -0,0 +1,32 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '', 'test_helper.rb')
+require 'rbbt/nlp/spaCy'
+require 'rbbt/document/corpus'
+
+class TestSpaCy < Test::Unit::TestCase
+  def _test_tokens
+    text = "I tell a story"
+
+    tokens = SpaCy.tokens(text)
+
+    assert_equal 4, tokens.length
+    assert_equal "tell", tokens[1].to_s
+  end
+
+  def test_segments
+    text = "I tell a story. It's a very good story."
+
+    corpus = Document::Corpus.setup({})
+
+    Document.setup(text, "TEST", "test_doc1", "simple_sentence")
+
+    corpus.add_document text
+    text.corpus = corpus
+
+    segments = SpaCy.segments(text)
+
+    segments.each do |segment|
+      assert_equal segment, segment.segid.tap{|e| e.corpus = corpus}.segment
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-text
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.3.3
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-13 00:00:00.000000000 Z
+date: 2020-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -78,6 +78,10 @@ files:
 - lib/rbbt/bow/bow.rb
 - lib/rbbt/bow/dictionary.rb
 - lib/rbbt/bow/misc.rb
+- lib/rbbt/document.rb
+- lib/rbbt/document/annotation.rb
+- lib/rbbt/document/corpus.rb
+- lib/rbbt/document/corpus/pubmed.rb
 - lib/rbbt/ner/NER.rb
 - lib/rbbt/ner/abner.rb
 - lib/rbbt/ner/banner.rb
@@ -98,18 +102,18 @@ files:
 - lib/rbbt/nlp/genia/sentence_splitter.rb
 - lib/rbbt/nlp/nlp.rb
 - lib/rbbt/nlp/open_nlp/sentence_splitter.rb
-- lib/rbbt/text/corpus.rb
-- lib/rbbt/text/corpus/document.rb
-- lib/rbbt/text/corpus/document_repo.rb
-- lib/rbbt/text/corpus/sources/pmid.rb
-- lib/rbbt/text/document.rb
-- lib/rbbt/text/segment.rb
-- lib/rbbt/text/segment/docid.rb
-- lib/rbbt/text/segment/named_entity.rb
-- lib/rbbt/text/segment/relationship.rb
-- lib/rbbt/text/segment/segmented.rb
-- lib/rbbt/text/segment/token.rb
-- lib/rbbt/text/segment/transformed.rb
+- lib/rbbt/nlp/spaCy.rb
+- lib/rbbt/segment.rb
+- lib/rbbt/segment/annotation.rb
+- lib/rbbt/segment/encoding.rb
+- lib/rbbt/segment/named_entity.rb
+- lib/rbbt/segment/overlaps.rb
+- lib/rbbt/segment/range_index.rb
+- lib/rbbt/segment/relationship.rb
+- lib/rbbt/segment/segmented.rb
+- lib/rbbt/segment/token.rb
+- lib/rbbt/segment/transformed.rb
+- lib/rbbt/segment/tsv.rb
 - share/install/software/ABNER
 - share/install/software/BANNER
 - share/install/software/ChemicalTagger
@@ -128,6 +132,9 @@ files:
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
+- test/rbbt/document/corpus/test_pubmed.rb
+- test/rbbt/document/test_annotation.rb
+- test/rbbt/document/test_corpus.rb
 - test/rbbt/entity/test_document.rb
 - test/rbbt/ner/test_NER.rb
 - test/rbbt/ner/test_abner.rb
@@ -146,16 +153,17 @@ files:
 - test/rbbt/nlp/genia/test_sentence_splitter.rb
 - test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
 - test/rbbt/nlp/test_nlp.rb
-- test/rbbt/text/corpus/sources/test_pmid.rb
-- test/rbbt/text/corpus/test_document.rb
-- test/rbbt/text/segment/test_named_entity.rb
-- test/rbbt/text/segment/test_relationship.rb
-- test/rbbt/text/segment/test_segmented.rb
-- test/rbbt/text/segment/test_transformed.rb
-- test/rbbt/text/test_corpus.rb
-- test/rbbt/text/test_document.rb
-- test/rbbt/text/test_segment.rb
+- test/rbbt/segment/test_annotation.rb
+- test/rbbt/segment/test_corpus.rb
+- test/rbbt/segment/test_encoding.rb
+- test/rbbt/segment/test_named_entity.rb
+- test/rbbt/segment/test_overlaps.rb
+- test/rbbt/segment/test_range_index.rb
+- test/rbbt/segment/test_transformed.rb
+- test/rbbt/test_document.rb
+- test/rbbt/test_segment.rb
 - test/test_helper.rb
+- test/test_spaCy.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses: []
 metadata: {}
@@ -182,18 +190,13 @@ test_files:
 - test/rbbt/nlp/test_nlp.rb
 - test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
 - test/rbbt/nlp/genia/test_sentence_splitter.rb
-- test/rbbt/text/test_document.rb
-- test/rbbt/text/corpus/sources/test_pmid.rb
-- test/rbbt/text/corpus/test_document.rb
-- test/rbbt/text/test_segment.rb
-- test/rbbt/text/test_corpus.rb
-- test/rbbt/text/segment/test_transformed.rb
-- test/rbbt/text/segment/test_relationship.rb
-- test/rbbt/text/segment/test_named_entity.rb
-- test/rbbt/text/segment/test_segmented.rb
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_misc.rb
 - test/rbbt/bow/test_dictionary.rb
+- test/rbbt/test_document.rb
+- test/rbbt/document/test_annotation.rb
+- test/rbbt/document/corpus/test_pubmed.rb
+- test/rbbt/document/test_corpus.rb
 - test/rbbt/entity/test_document.rb
 - test/rbbt/ner/test_patterns.rb
 - test/rbbt/ner/test_NER.rb
@@ -209,4 +212,13 @@ test_files:
 - test/rbbt/ner/test_finder.rb
 - test/rbbt/ner/test_linnaeus.rb
 - test/rbbt/ner/test_oscar4.rb
+- test/rbbt/test_segment.rb
+- test/rbbt/segment/test_transformed.rb
+- test/rbbt/segment/test_overlaps.rb
+- test/rbbt/segment/test_annotation.rb
+- test/rbbt/segment/test_named_entity.rb
+- test/rbbt/segment/test_encoding.rb
+- test/rbbt/segment/test_range_index.rb
+- test/rbbt/segment/test_corpus.rb
+- test/test_spaCy.rb
 - test/test_helper.rb

data/lib/rbbt/text/corpus.rb

@@ -1,106 +0,0 @@
-require 'rbbt/text/corpus/document'
-require 'rbbt/text/corpus/document_repo'
-
-class Corpus
-  class << self
-    attr_accessor :claims
-    def claim(namespace, &block)
-      @@claims = {}
-      @@claims[namespace] = block
-    end
-
-  end
-  attr_accessor :corpora_path, :document_repo, :persistence_dir, :global_annotations
-
-  def initialize(corpora_path = nil)
-    @corpora_path = case
-                   when corpora_path.nil?
-                     Rbbt.corpora
-                   when (not Path === corpora_path)
-                     Path.setup(corpora_path)
-                   else
-                     corpora_path
-                   end
-
-    @corpora_path = @corpora_path.find
-    @persistence_dir = File.join(@corpora_path, "annotations")
-
-    Misc.lock(@persistence_dir) do
-      @global_annotations = TSV.setup(Persist.open_tokyocabinet(File.join(@persistence_dir, "global_annotations"), false, :list), :key => "ID", :fields => ["Start", "End", "JSON", "Document ID", "Entity Type"])
-      @global_annotations.unnamed = true
-      @global_annotations.close
-    end
- 
-    Misc.lock(@corpora_path.document_repo) do
-      @document_repo   = DocumentRepo.open_tokyocabinet @corpora_path.document_repo, false
-      @document_repo.close
-    end
-
- end
-
-  def persistence_for(docid)
-    File.join(persistence_dir, docid)
-  end
-
-
-  def docid(docid)
-    begin
-      if @document_repo.include?(docid)
-        Document.new(persistence_for(docid), docid, @document_repo[docid], @global_annotations, self)
-      else
-        namespace, id, type = docid.split(":")
-        if @@claims.include?(namespace)
-
-          docid = self.instance_exec id, type, &(@@claims[namespace])
-          docid = docid.first if Array === docid
-          self.docid(docid)
-        else
-          raise "Document '#{ docid }' was not found." unless @document_repo.include? docid
-        end
-      end
-    ensure
-      @document_repo.close
-    end
-  end
-
-  def document(namespace, id, type, hash)
-    docid = [namespace, id, type, hash] * ":"
-    self.docid(docid)
-  end
-
-  def add_document(text, namespace = nil, id = nil, type = nil)
-    text = Misc.fixutf8(text)
-    hash = Digest::MD5.hexdigest(text)
-    @document_repo.add(text, namespace, id, type, hash)
-  end
-
-  def add_docid(text, docid)
-    namespace, id, type, hash = docid.split(":")
-    @document_repo.add(text, namespace, id, type, hash)
-  end
-
-
-  def find(namespace=nil, id = nil, type = nil, hash = nil)
-    @document_repo.find(namespace, id, type, hash).collect{|docid|
-      self.docid(docid)
-    }
-  end
-
-  def find_docid(docid)
-    @document_repo.find_docid(docid).collect{|docid|
-      self.docid(docid)
-    }
-  end
-
-  def exists?(namespace=nil, id = nil, type = nil, hash = nil)
-    find(namespace, id, type, hash).any?
-  end
-
-  def [](docid)
-    self.docid(docid)
-  end
-  
-  def include?(id)
-    @document_repo.include? id
-  end
-end