rbbt-text 1.1.9 → 1.3.3

data/lib/rbbt/text/segment/token.rb

@@ -1,49 +0,0 @@
-require 'rbbt/annotations'
-require 'rbbt/text/segment'
-
-module Token
-  attr_accessor :offset, :original
-  
-  def self.all_annotations
-    [:offset, :original]
-  end
-
-  def self.setup(text, start, original = nil)
-    text.extend Token
-    text.offset = start
-    text.original = original
-    text
-  end
-  
-  def info
-    {:original => original, :offset => offset}
-  end
-
-  def id
-    Misc.hash2md5 info.merge :self => self
-  end
-
-  def end
-    offset + self.length - 1
-  end
-
-  def range
-    (offset..self.end)
-  end
-
-  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
-
-    tokens = []
-    while matchdata = text.match(split_at)
-      tokens << Token.setup(matchdata.pre_match, start) unless matchdata.pre_match.empty?
-      tokens << Token.setup(matchdata.captures.first, start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
-      start += matchdata.end(0)
-      text = matchdata.post_match
-    end
-
-    tokens << Token.setup(text, start) unless text.empty?
-
-    tokens
-  end
-end
-

data/test/rbbt/text/corpus/sources/test_pmid.rb

@@ -1,33 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../../..', 'test_helper.rb')
-require 'rbbt/text/document'
-require 'rbbt/text/corpus'
-require 'rbbt/text/corpus/sources/pmid'
-
-class TestCorpusPMID < Test::Unit::TestCase
-  def setup
-    Log.severity = 0
-    Document.corpus = Corpus.new Rbbt.tmp.test.document_corpus
-
-    Corpus::Document.define :words do
-      words = self.text.split(" ")
-      Segment.align(self.text, words)
-    end
-
-    Corpus::Document.define :genes do
-      require 'rbbt/ner/banner'
-      Banner.new.match(self.text)
-    end
-
-    Corpus::Document.persist_in_global_tsv("genes")
-    Corpus::Document.persist_in_global_tsv(:words)
-  end
-
-  def test_query
-    docids = Document.corpus.add_pubmed_query("SARS-Cov-2", 2000, :abstract)
-
-    docids.each do |docid|
-      iif Document.corpus.docid(docid).text
-    end
-  end
-end
-

data/test/rbbt/text/corpus/test_document.rb

@@ -1,52 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/text/corpus/document'
-
-class TestCorpusDocument < Test::Unit::TestCase
-  def setup
-    Log.severity = 0
-
-    Corpus::Document.define :words do
-      words = self.text.split(" ")
-      Segment.align(self.text, words)
-    end
-
-    Open.mkdir Rbbt.tmp.test.annotations.find
-    Corpus::Document.persist_in_global_tsv(:words, Rbbt.tmp.test.anotations.words.find)
-
-
-    Corpus::Document.define_multiple :words2 do |documents|
-      documents.collect do |doc|
-        words = doc.text.split(" ")
-        Segment.align(doc.text, words)
-      end
-    end
-
-    Corpus::Document.persist_in_global_tsv(:words2, Rbbt.tmp.test.anotations.counts.find)
-  end
-
-  def test_words
-    text = "This is a test document"
-    document = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc", text)
-    assert_equal Segment.sort(document.words), text.split(" ")
-  end
-
-  def test_words_multiple
-    document1 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:1", "This is a test document")
-    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is a another test document")
-
-    docs = [document1, document2]
-
-    Corpus::Document.prepare_multiple(docs, :words2)
-    
-    assert_equal document1.words2, document1.text.split(" ")
-    assert_equal document2.words2, document2.text.split(" ")
-
-    document1 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:1", "This is a test document")
-    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is a another test document")
-
-    docs = [document1, document2]
-
-    Corpus::Document.prepare_multiple(docs, :words2)
-  end
-end
-

data/test/rbbt/text/segment/test_segmented.rb

@@ -1,23 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/text/segment/segmented'
-
-class TestClass < Test::Unit::TestCase
-  def test_split
-    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
-
-    gene1 = "TP53"
-    gene1.extend Segment
-    gene1.offset = a.index gene1
-
-    gene2 = "CDK5R1"
-    gene2.extend Segment
-    gene2.offset = a.index gene2
-
-    gene3 = "TP53 gene"
-    gene3.extend Segment
-    gene3.offset = a.index gene3
-
-    Segmented.setup(a, [gene2, gene1, gene3])
-    assert_equal ["This sentence mentions the ", gene3, " and the ", gene2, " protein"], a.split_segments
-  end
-end

data/test/rbbt/text/test_corpus.rb

@@ -1,34 +0,0 @@
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'test/unit'
-require 'rbbt-util'
-require 'rbbt/text/corpus'
-
-class Corpus::Document
-
-  define :words do
-    text.split(" ")
-  end
-end
-
-class TestClass < Test::Unit::TestCase
-  def test_document
-    Log.severity = 0
-    text = "This is a test document"
-
-    docid = nil
-    TmpFile.with_file do |dir|
-      corpus = Corpus.new dir
-      docid = corpus.add_document text, :TEST, :test_doc
-      document = corpus.docid(docid)
-      assert_equal text, document.text
-
-      corpus = Corpus.new dir
-      document = corpus.docid(docid)
-      assert_equal text, document.text
-      document = corpus.find(:TEST, :test_doc).first
-      assert_equal text, document.text
-    end
-  end
-end
-

data/test/rbbt/text/test_document.rb

@@ -1,58 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
-require 'rbbt/text/document'
-require 'rbbt/text/corpus/sources/pmid'
-
-class TestDocument < Test::Unit::TestCase
-  def setup
-    Log.severity = 0
-    Document.corpus = Corpus.new Rbbt.tmp.test.document_corpus
-
-    Corpus::Document.define :words do
-      words = self.text.split(" ")
-      Segment.align(self.text, words)
-    end
-
-    Corpus::Document.define :genes do
-      require 'rbbt/ner/banner'
-      Banner.new.match(self.text)
-    end
-
-    Corpus::Document.persist_in_global_tsv("genes")
-    Corpus::Document.persist_in_global_tsv(:words)
-  end
-
-  def test_title_and_text
-    document = Document.setup('PMID:32272262')
-
-    assert document.text.downcase.include?("covid")
-    assert_equal "High-resolution Chest CT Features and Clinical Characteristics of Patients Infected with COVID-19 in Jiangsu, China.", document.title
-  end
-
-  def test_full_text
-    document = Document.setup('PMID:4304705')
-    assert document.text.length < document.full_text.length
-  end
-
-  def test_words
-    document = Document.setup('PMID:32272262')
-    words = document.entities :words
-    assert words.first.respond_to?(:offset)
-  end
-
-  def test_genes
-    text = "This is a mention to TP53, a gene that should be found"
-    document = Document.setup(Document.corpus.add_document(text, "TEST"))
-    genes = document.entities :genes
-
-    assert_equal "TP53", genes.first
-    assert genes.first.respond_to?(:offset)
-
-    text = "This is a mention to TP53, a gene that should be found"
-    document = Document.setup(Document.corpus.add_document(text, "TEST"))
-    genes = document.entities :genes
-
-    assert_equal "TP53", genes.first
-    assert genes.first.respond_to?(:offset)
-  end
-end
-

data/test/rbbt/text/test_segment.rb

@@ -1,100 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
-require 'rbbt/text/segment'
-
-class TestSegment < Test::Unit::TestCase
-  def test_info
-    a = "test"
-    a.extend Segment
-    a.offset = 10
-    assert a.info.include? :offset
-  end
-
-  def test_sort
-    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
-
-    gene1 = "TP53"
-    gene1.extend Segment
-    gene1.offset = a.index gene1
-
-    gene2 = "CDK5R1"
-    gene2.extend Segment
-    gene2.offset = a.index gene2
-
-    assert_equal [gene1,gene2], Segment.sort([gene2,gene1])
-  end
-
-  def test_clean_sort
-    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
-
-    gene1 = "TP53"
-    gene1.extend Segment
-    gene1.offset = a.index gene1
-
-    gene2 = "CDK5R1"
-    gene2.extend Segment
-    gene2.offset = a.index gene2
-
-    gene3 = "TP53 gene"
-    gene3.extend Segment
-    gene3.offset = a.index gene3
-
-    assert_equal [gene3,gene2], Segment.clean_sort([gene2,gene1,gene3])
-  end
-
-  def test_split
-    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
-
-    gene1 = "TP53"
-    gene1.extend Segment
-    gene1.offset = a.index gene1
-
-    gene2 = "CDK5R1"
-    gene2.extend Segment
-    gene2.offset = a.index gene2
-
-    gene3 = "TP53 gene"
-    gene3.extend Segment
-    gene3.offset = a.index gene3
-
-    assert_equal ["This sentence mentions the ", gene3, " and the ", gene2, " protein"], Segment.split(a, [gene2,gene1,gene3])
-  end
-
-
-  def test_align
-    text =<<-EOF
-Atypical teratoid/rhabdoid tumors (AT/RTs) are highly aggressive brain tumors of early childhood poorly responding to therapy.
-    EOF
-
-    parts = text.split(/\W/)
-    Segment.align(text, parts)
-
-    assert_equal "Atypical teratoid/".length, parts.select{|s| s == "rhabdoid"}.first.offset
-  end
-
-  def test_segment_index
-    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
-
-    gene1 = "TP53"
-    gene1.extend Segment
-    gene1.offset = a.index gene1
-
-    gene2 = "CDK5R1"
-    gene2.extend Segment
-    gene2.offset = a.index gene2
-
-    gene3 = "TP53 gene"
-    gene3.extend Segment
-    gene3.offset = a.index gene3
-
-    index = Segment.index([gene1, gene2, gene3])
-    assert_equal %w(CDK5R1), index[gene2.offset + 1]
-
-    TmpFile.with_file do |fwt|
-      index = Segment.index([gene1, gene2, gene3], fwt)
-      assert_equal %w(CDK5R1), index[gene2.offset + 1]
-      index = Segment.index([gene1, gene2, gene3], fwt)
-      assert_equal %w(CDK5R1), index[gene2.offset + 1]
-    end
-  end
-end
-