rbbt-text 1.2.0 → 1.3.0

data/lib/rbbt/segment/range_index.rb

@@ -0,0 +1,35 @@
+module Segment::RangeIndex
+  attr_accessor :corpus
+
+  def [](*args)
+    res = super(*args)
+    SegID.setup(res, :corpus => corpus)
+  end
+
+  def self.index(segments, corpus, persist_file = :memory)
+    segments = segments.values.flatten if Hash === segments
+
+    annotation_index = 
+      Persist.persist("Segment_index", :fwt, :persist => (! (persist_file.nil? or persist_file == :memory)), :file => persist_file) do
+
+        value_size = 0
+        index_data = segments.collect{|segment| 
+          next if segment.offset.nil?
+          range = segment.range
+          value_size = [segment.segid.length, value_size].max
+          [segment.segid, [range.begin, range.end]]
+        }.compact
+
+        fwt = FixWidthTable.get :memory, value_size, true
+        fwt.add_range index_data
+
+        fwt
+      end
+
+    annotation_index.extend Segment::RangeIndex
+    annotation_index.corpus = corpus
+    annotation_index
+  end
+
+end
+  

data/lib/rbbt/{text/segment → segment}/segmented.rb

@@ -1,5 +1,5 @@
 require 'rbbt/annotations'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 
 module Segmented
   extend Annotation

data/lib/rbbt/segment/token.rb

@@ -0,0 +1,23 @@
+require 'rbbt/segment'
+
+module Token
+  extend Entity
+  include Segment
+
+  self.annotation :original
+
+  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
+
+    tokens = []
+    while matchdata = text.match(split_at)
+      tokens << Token.setup(matchdata.pre_match, :offset => start) unless matchdata.pre_match.empty?
+      tokens << Token.setup(matchdata.captures.first, :offset => start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      start += matchdata.end(0)
+      text = matchdata.post_match
+    end
+
+    tokens << Token.setup(text, :offset => start) unless text.empty?
+
+    tokens
+  end
+end

data/lib/rbbt/{text/segment → segment}/transformed.rb

@@ -1,6 +1,3 @@
-require 'rbbt/util/misc'
-require 'rbbt/text/segment'
-
 module Transformed
 
   def self.transform(text, segments, replacement = nil, &block)
@@ -111,10 +108,10 @@ module Transformed
 
       self[updated_begin..updated_end] = new
 
-      @transformed_segments[segment.segment_id] = [segment.range, diff, updated_text, updated_range, @transformed_segments.size]
+      @transformed_segments[segment.object_id] = [segment.range, diff, updated_text, updated_range, @transformed_segments.size]
 
       segment.replace original_text
-      stack << segment.segment_id
+      stack << segment.object_id
     end
     @transformation_stack << stack
   end
@@ -122,13 +119,13 @@ module Transformed
   def fix_segment(segment, range, diff)
     case
       # Before
-    when segment.end < range.begin
+    when segment.eend < range.begin
       # After
     when segment.offset.to_i > range.end + diff
       segment.offset = segment.offset.to_i - diff
       # Includes
-    when (segment.offset.to_i <= range.begin and segment.end >= range.end + diff)
-      segment.replace self[segment.offset.to_i..segment.end - diff]
+    when (segment.offset.to_i <= range.begin and segment.eend >= range.end + diff)
+      segment.replace self[segment.offset.to_i..segment.eend - diff]
     else
       raise "Segment Overlaps"
     end
@@ -141,7 +138,8 @@ module Transformed
 
     if first_only
       @transformation_stack.pop.reverse.each do |id|
-        orig_range, diff, text, range = @transformed_segments.delete id
+        segment_info = @transformed_segments.delete id
+        orig_range, diff, text, range = segment_info 
 
         new_range = (range.begin..range.last + diff)
         self[new_range] = text

data/lib/rbbt/segment/tsv.rb

@@ -0,0 +1,41 @@
+#module Segment
+#
+#  def self.set_tsv_fields(fields, segments)
+#    tsv_fields = []
+#    add_types = ! (fields.delete(:no_types) || fields.delete("no_types") || fields.include?(:JSON) || fields.include?("JSON"))
+#    literal = (fields.delete(:literal) || fields.delete("literal"))
+#    tsv_fields << "Start" << "End"
+#    tsv_fields << :annotation_types if add_types
+#    tsv_fields << :literal if literal
+#
+#    if fields.any? and not (fields == [:all] or fields == ["all"])
+#      tsv_fields.concat fields 
+#    else
+#      tsv_fields.concat segments.first.annotations if segments.any?
+#    end
+#    tsv_fields
+#    tsv_fields.collect!{|f| f.to_s}
+#    tsv_fields.delete "offset"
+#    tsv_fields
+#  end
+#
+#  def self.tsv(segments, *fields)
+#    fields = set_tsv_fields fields, segments
+#    tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :double)
+#
+#    segments.each do |segment|
+#      tsv[segment.segment_id] = self.tsv_values_for_segment(segment, fields)
+#    end
+#
+#    tsv
+#  end
+#
+#  def self.load_tsv(tsv)
+#    fields = tsv.fields
+#    tsv.with_unnamed do
+#      tsv.collect do |id, values|
+#        Annotated.load_tsv_values(id, values, fields)
+#      end
+#    end
+#  end
+#end

data/share/install/software/Linnaeus

@@ -12,7 +12,7 @@ pkg_dir="`opt_dir \"$name\"`"
 build_dir=`build_dir`
 mv "$build_dir" "$pkg_dir"
 tmp_file="~/.rbbt/tmp/species-proxy-properties.tmp"
-mkdir -p $(basename "$tmp_file")
+mkdir -p $(dirname "$tmp_file")
 cat  "$pkg_dir/species-proxy/properties.conf" |grep -v "^.dir =" >> $tmp_file
 echo "\$dir = $pkg_dir/species-proxy/" > "$pkg_dir/species-proxy/properties.conf"
 cat $tmp_file | grep -v "^#" >>  "$pkg_dir/species-proxy/properties.conf"

data/test/rbbt/document/corpus/test_pubmed.rb

@@ -0,0 +1,15 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/document'
+require 'rbbt/document/corpus'
+require 'rbbt/document/corpus/pubmed'
+
+class TestCorpusPubmed < Test::Unit::TestCase
+  def test_add_pmid
+    corpus = Document::Corpus.setup({})
+
+    document = corpus.add_pmid("32299157", :abstract).first
+    title = document.to(:title)
+    assert title.include?("COVID-19")
+  end
+end
+

data/test/rbbt/document/test_annotation.rb

@@ -0,0 +1,140 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/document'
+require 'rbbt/document/corpus'
+require 'rbbt/segment'
+require 'rbbt/document/annotation'
+require 'rbbt/segment/named_entity'
+
+class TestAnnotation < Test::Unit::TestCase
+  class CalledOnce < Exception; end
+  def setup
+    Document.define :words do
+      self.split(" ")
+    end
+
+    $called_once = false
+    Document.define :persisted_words do
+      raise CalledOnce if $called_once
+      $called_once = true
+      self.split(" ")
+    end
+
+    Document.define_multiple :multiple_words do |list|
+      list.collect{|doc| doc.words}
+    end
+
+    Document.define :ner do
+      $called_once = true
+      self.split(" ").collect{|e| NamedEntity.setup(e, :code => Misc.digest(e)) }
+    end
+
+    Document.persist :ner
+  end
+
+  def test_define
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    corpus.extend Document::Corpus
+
+    corpus.add_document(text)
+
+    assert_equal text[text.words[1].range], text.words[1]
+  end
+
+  def test_define_multiple
+    text1 = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    text2 = "This is another sentence"
+    Document.setup(text1, "TEST", "test_doc1", nil)
+    Document.setup(text2, "TEST", "test_doc2", nil)
+
+    corpus = {}
+    corpus.extend Document::Corpus
+
+    corpus.add_document(text1)
+    corpus.add_document(text2)
+
+    assert_equal 2, Document.setup([text1, text2]).multiple_words.length
+    assert_equal text1.split(" "), text1.multiple_words
+
+    #Document.persist :multiple_words, :annotations, :annotation_repo => Rbbt.tmp.test.multiple_words
+    #assert_equal 2, Document.setup([text1, text2]).multiple_words.length
+    #assert_equal text1.split(" "), text1.multiple_words
+  end
+
+  def test_persist
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    corpus.extend Document::Corpus
+
+    corpus.add_document(text)
+
+    assert_equal "persisted_words", text.persisted_words.first.type
+
+    assert_raise CalledOnce do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+
+    Log.severity = 0
+    Document.persist :persisted_words, :annotations, :file => Rbbt.tmp.test.persisted_words.find(:user)
+
+    $called_once = false
+    text.persisted_words
+    assert $called_once
+
+    assert_nothing_raised  do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+  end
+
+  def test_persist_annotation_repo
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    corpus.extend Document::Corpus
+
+    corpus.add_document(text)
+
+    assert_equal "persisted_words", text.persisted_words.first.type
+
+    assert_raise CalledOnce do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+
+    Log.severity = 0
+    Document.persist :persisted_words, :annotations, :annotation_repo => Rbbt.tmp.test.persisted_words_repo.find(:user)
+
+    $called_once = false
+    text.persisted_words
+    assert $called_once
+
+    assert_nothing_raised  do
+      assert_equal text[text.persisted_words[1].range], text.persisted_words[1]
+    end
+  end
+
+  def test_persist_ner
+    text = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = {}
+    corpus.extend Document::Corpus
+
+    corpus.add_document(text)
+
+
+    text.ner
+
+    $called_once = false
+    text.ner
+
+    assert ! $called_once
+    
+    assert  text.ner.first.segid.include?("TEST:")
+  end
+end
+

data/test/rbbt/document/test_corpus.rb

@@ -0,0 +1,33 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/document'
+require 'rbbt/document/corpus'
+
+class TestDocumentCorpus < Test::Unit::TestCase
+  def test_corpus
+    text = "This is a document"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    corpus = Document::Corpus.setup({})
+
+    corpus.add_document(text)
+
+    docid = text.docid(corpus)
+
+    assert_equal docid.document, text
+  end
+
+  def test_find
+    text = "This is a document"
+    Document.setup(text, "TEST", "test_doc1", nil)
+
+    TmpFile.with_file do |path|
+      corpus = Persist.open_tokyocabinet(path, true, :single, "BDB")
+      corpus.extend Document::Corpus
+
+      corpus.add_document(text)
+
+      assert corpus.prefix("TEST:").include?(text.docid)
+    end
+  end
+end
+

data/test/rbbt/ner/test_finder.rb

@@ -8,13 +8,13 @@ require 'rbbt/sources/NCI'
 
 class TestFinder < Test::Unit::TestCase
 
-  def test_namespace_and_format
+  def _test_namespace_and_format
     f = Finder.new(CMD.cmd("head -n 1000", :in => Open.open(Organism.identifiers(Organism.default_code("Hsa")).produce.find)))
     assert_equal Organism.default_code("Hsa"), f.instances.first.namespace
     assert_equal "Ensembl Gene ID", f.instances.first.format
   end
 
-  def test_find
+  def _test_find
     f = Finder.new(Organism.lexicon(Organism.default_code("Hsa")), :grep => ["SF3B1"])
 
     assert_equal "ENSG00000115524", f.find("SF3B1").first
@@ -23,7 +23,7 @@ class TestFinder < Test::Unit::TestCase
     end
   end
 
-  def test_find2
+  def _test_find2
     f = Finder.new(Organism.lexicon(Organism.default_code("Hsa")), :grep => ["RASGRF2"])
 
     m = f.find("RAS").first

data/test/rbbt/ner/test_g_norm_plus.rb

@@ -9,7 +9,17 @@ We found that TP53 is regulated by MDM2 in Homo sapiens
     EOF
 
     mentions = GNormPlus.process({:file => text})
-    Log.tsv mentions
+    assert_equal 1, mentions.length
+    assert_equal 2, mentions["file"].length
+  end
+
+  def test_entities
+    text =<<-EOF
+We found that TP53 is regulated by MDM2 in Homo sapiens
+    EOF
+
+    mentions = GNormPlus.entities({:file => text})
+    mentions["file"].include? "TP53"
   end
 end
 

data/test/rbbt/ner/test_patterns.rb

@@ -2,17 +2,17 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.r
 require 'rbbt/ner/patterns'
 
 class TestPatternRelExt < Test::Unit::TestCase
-  def test_simple_pattern
+  def _test_simple_pattern
     text = "Experiments have shown that TP53 interacts with CDK5 under certain conditions"
 
     gene1 = "TP53"
-    NamedEntity.setup(gene1, text.index(gene1), "Gene")
+    NamedEntity.setup(gene1, :offset => text.index(gene1), :entity_type => "Gene")
 
     gene2 = "CDK5"
-    NamedEntity.setup(gene2, text.index(gene2), "Gene")
+    NamedEntity.setup(gene2, :offset => text.index(gene2), :entity_type => "Gene")
 
     interaction = "interacts"
-    NamedEntity.setup(interaction, text.index(interaction), "Interaction")
+    NamedEntity.setup(interaction, :offset => text.index(interaction), :entity_type => "Interaction")
 
     Segmented.setup(text, [gene1, gene2, interaction])
 
@@ -23,13 +23,13 @@ class TestPatternRelExt < Test::Unit::TestCase
     text = "Experiments have shown that TP53 found in cultivated cells interacts with CDK5 under certain conditions"
 
     gene1 = "TP53"
-    NamedEntity.setup(gene1, text.index(gene1), "Gene")
+    NamedEntity.setup(gene1, :offset => text.index(gene1), :entity_type => "Gene")
 
     gene2 = "CDK5"
-    NamedEntity.setup(gene2, text.index(gene2), "Gene")
+    NamedEntity.setup(gene2, :offset => text.index(gene2), :entity_type => "Gene")
 
     interaction = "interacts"
-    NamedEntity.setup(interaction, text.index(interaction), "Interaction")
+    NamedEntity.setup(interaction, :offset => text.index(interaction), :entity_type => "Interaction")
 
     Segmented.setup(text, {:entities => [gene1, gene2, interaction]})
 
@@ -40,7 +40,7 @@ class TestPatternRelExt < Test::Unit::TestCase
       PatternRelExt.new(["NP[entity:Gene] VP[stem:interacts] with NP[entity:Gene]"]).match_sentences([text]).first.first
   end
 
-  def test_chunk_pattern
+  def _test_chunk_pattern
     text = "There is a concern with the use of thiazolidinediones in patients with an increased risk of colon cancer (e.g., familial colon polyposis)."
 
     drug = "thiazolidinediones"
@@ -57,7 +57,7 @@ class TestPatternRelExt < Test::Unit::TestCase
   end
 
 
-  def test_entities_with_spaces
+  def _test_entities_with_spaces
     PatternRelExt.new("NP[entity:Gene Name]").token_trie
   end