rbbt-text 0.2.1 → 0.5.0

data/test/rbbt/ner/annotations/test_named_entity.rb

@@ -0,0 +1,14 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/ner/annotations'
+require 'rbbt/ner/annotations/named_entity'
+require 'rbbt/ner/annotations/transformed'
+
+class TestClass < Test::Unit::TestCase
+  def test_info
+    a = "test"
+    a.extend NamedEntity
+    assert(! a.info.keys.include?("offset"))
+    a.offset = 10
+    assert a.info.keys.include? "offset"
+  end
+end

data/test/rbbt/ner/annotations/test_transformed.rb

@@ -0,0 +1,175 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/ner/annotations/transformed'
+require 'rbbt/ner/annotations/named_entity'
+
+class TestClass < Test::Unit::TestCase
+  def test_transform
+    a = "This sentence mentions the TP53 gene and the CDK5 protein"
+    original = a.dup
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+
+    gene2 = "CDK5"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+
+    assert_equal gene1, a[gene1.range]
+    assert_equal gene2, a[gene2.range]
+    
+    c = a.dup
+
+    c[gene2.range] = "GN"
+    assert_equal c, Transformed.transform(a,[gene2], "GN")
+    c[gene1.range] = "GN"
+    assert_equal c, Transformed.transform(a,[gene1], "GN")
+
+    assert_equal gene2.offset, a.transformation_offset_differences.first.first.first
+    assert_equal gene1.offset, a.transformation_offset_differences.last.first.first
+
+
+    gene3 = "GN gene"
+    gene3.extend NamedEntity
+    gene3.offset = a.index gene3
+
+    assert_equal gene3, a[gene3.range]
+
+    a.restore([gene3])
+    assert_equal original, a
+    assert_equal "TP53 gene", a[gene3.range]
+
+  end
+
+  def test_with_transform
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    original = a.dup
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+
+    gene2 = "CDK5R1"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+
+    Transformed.with_transform(a, [gene1], "GN") do 
+      assert_equal original.sub("TP53", 'GN'), a
+    end
+    assert_equal original, a
+
+    Transformed.with_transform(a, [gene1,gene2], "GN") do 
+      assert_equal original.gsub(/TP53|CDK5R1/, 'GN'), a
+    end
+    assert_equal original, a
+
+    Transformed.with_transform(a, [gene1], "GN") do 
+      Transformed.with_transform(a, [gene2], "GN") do 
+        assert_equal original.gsub(/TP53|CDK5R1/, 'GN'), a
+      end
+      assert_equal original.gsub(/TP53/, 'GN'), a
+    end
+    assert_equal original, a
+
+    exp1, exp2 = nil, nil
+    expanded_genes = Transformed.with_transform(a, [gene1,gene2], "GN") do 
+      exp1 = "GN gene"
+      exp1.extend NamedEntity
+      exp1.offset = a.index exp1
+      exp2 = "GN protein"
+      exp2.extend NamedEntity
+      exp2.offset = a.index exp2
+
+      [exp1, exp2]
+    end
+    assert_equal original, a
+
+    assert_equal "TP53 gene", exp1
+    assert_equal "CDK5R1 protein", exp2
+  end
+
+  def test_html
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "CDK5R1"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Protein"
+
+    Transformed.with_transform(a, [gene1,gene2], Proc.new{|e| e.html}) do 
+      assert_equal "This sentence mentions the <span class='Entity' attr-entity-type='Gene'>TP53</span> gene and the <span class='Entity' attr-entity-type='Protein'>CDK5R1</span> protein", a
+    end
+  end
+
+  def test_html_with_offset
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+    Segment.annotate(a, 10)
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1 
+    gene1.offset += 10
+    gene1.type = "Gene"
+
+    gene2 = "CDK5R1"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.offset += 10
+    gene2.type = "Protein"
+
+    Transformed.with_transform(a, [gene1,gene2], Proc.new{|e| e.html}) do 
+      assert_equal "This sentence mentions the <span class='Entity' attr-entity-type='Gene'>TP53</span> gene and the <span class='Entity' attr-entity-type='Protein'>CDK5R1</span> protein", a
+    end
+  end
+
+  def test_overlap
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "TP53 gene"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Expanded Gene"
+
+    assert_equal [gene1], Segment.overlaps(Segment.sort([gene1,gene2]))
+
+    Transformed.with_transform(a, [gene1], Proc.new{|e| e.html}) do 
+      assert_equal "This sentence mentions the <span class='Entity' attr-entity-type='Gene'>TP53</span> gene and the CDK5R1 protein", a
+      Transformed.with_transform(a, [gene2], Proc.new{|e| e.html}) do 
+        assert_equal "This sentence mentions the <span class='Entity' attr-entity-type='Expanded Gene'><span class='Entity' attr-entity-type='Gene'>TP53</span> gene</span> and the CDK5R1 protein", a
+      end
+    end
+  end
+
+  def test_cascade_with_overlap_ignored
+    a = "This sentence mentions the HDL-C gene and the CDK5R1 protein"
+
+    gene1 = "HDL-C"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "-"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Dash"
+
+    Transformed.with_transform(a, [gene1], Proc.new{|e| e.html}) do 
+      one = a.dup
+      Transformed.with_transform(a, [gene2], Proc.new{|e| e.html}) do 
+        assert_equal one, a
+      end
+    end
+
+  end
+end
+

data/test/rbbt/ner/test_abner.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../../test_helper'
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
 require 'rbbt/ner/abner'
 require 'test/unit'
 

data/test/rbbt/ner/test_annotations.rb

@@ -1,8 +1,70 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/ner/annotations'
+require 'rbbt/ner/annotations/named_entity'
+require 'rbbt/ner/annotations/transformed'
 
 class TestClass < Test::Unit::TestCase
-  def test_true
-    assert true
+  def test_info
+    a = "test"
+    a.extend NamedEntity
+    a.type = "type"
+    assert a.info.keys.include? "type"
+  end
+
+  def test_segment_type
+    a = "test"
+    a.extend NamedEntity
+    assert a.segment_types.include? "NamedEntity"
+  end
+
+  def test_align
+    text =<<-EOF
+Atypical teratoid/rhabdoid tumors (AT/RTs) are highly aggressive brain tumors of early childhood poorly responding to therapy.
+    EOF
+
+    parts = text.split(/\W/)
+    Segment.align(text, parts)
+
+    assert_equal "Atypical teratoid/".length, parts.select{|s| s == "rhabdoid"}.first.offset
+  end
+
+  def test_sort
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "CDK5R1"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Gene"
+
+    assert_equal [gene1,gene2], Segment.sort([gene2,gene1])
+
+  end
+
+  def test_clean_sort
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    gene2 = "CDK5R1"
+    gene2.extend NamedEntity
+    gene2.offset = a.index gene2
+    gene2.type = "Gene"
+
+    gene3 = "TP53 gene"
+    gene3.extend NamedEntity
+    gene3.offset = a.index gene3
+    gene3.type = "Gene"
+
+    assert_equal [gene3,gene2], Segment.clean_sort([gene2,gene1,gene3])
+
   end
 end
 

data/test/rbbt/ner/test_banner.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../../test_helper'
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
 require 'rbbt/ner/banner'
 require 'test/unit'
 

data/test/rbbt/ner/test_chemical_tagger.rb

@@ -0,0 +1,56 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
+require 'rbbt/ner/chemical_tagger'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+class TestChemicalTagger < Test::Unit::TestCase
+
+  def test_match
+    begin
+      ner = ChemicalTagger.new
+      str  = "Alternatively, rearrangement of O-(w-haloalkyl)esters 34 of 2-carboethoxy-N-hydroxypyridine-2-selone affords azonianaphthalenium halides 37 in 79% yield"
+      mentions = ner.match(str, "CM", false)
+      
+      good_mentions = ["2-carboethoxy-N-hydroxypyridine-2-selone", "O-(w-haloalkyl)esters"]
+
+      good_mentions.each{|mention| 
+        assert(mentions.include? mention)
+      }
+    rescue
+      puts $!.message
+      puts $!.backtrace
+    end
+  end
+
+  def test_ranges
+    begin
+      ner = ChemicalTagger.new
+      str =<<-EOF 
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This otherone talks about O-(w-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
+
+This otherone talks about O-(w-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
+      EOF
+      
+      mentions = ner.match(str, "CM", false)
+
+      str_original = str.dup
+      mentions.each do |mention|
+        str[mention.range] = mention
+      end
+
+      assert_equal str_original, str
+
+    rescue
+      puts $!.message
+      puts $!.backtrace
+    end
+  end
+
+end

data/test/rbbt/ner/test_ngram_prefix_dictionary.rb

@@ -0,0 +1,20 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/ner/ngram_prefix_dictionary'
+require 'rbbt/util/tmpfile'
+
+class TestNGramPrefixDictionary < Test::Unit::TestCase
+  def test_match
+    lexicon =<<-EOF
+C1;aa;AA;bb b
+C2;11;22;3 3;bb
+    EOF
+
+    TmpFile.with_file(lexicon) do |file|
+      index = NGramPrefixDictionary.new(TSV.new(file, :flat, :sep => ';'), "test")
+
+      matches = index.match(' asdfa dsf asdf aa asdfasdf ')
+      assert matches.select{|m| m.code.include? 'C1'}.any?
+    end
+  end
+end
+

data/test/rbbt/ner/{test_oscar3.rb → test_oscar4.rb}

@@ -1,18 +1,17 @@
-require File.dirname(__FILE__) + '/../../test_helper'
-require 'rbbt/ner/oscar3'
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
+require 'rbbt/ner/oscar4'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-class TestOSCAR3 < Test::Unit::TestCase
-
+class TestOSCAR4 < Test::Unit::TestCase
 
   def test_match
     begin
-      ner = OSCAR3.new
-      str  = "Alternatively, rearrangement of O-(ω-haloalkyl)esters 34 of 2-carboethoxy-N-hydroxypyridine-2-selone affords azonianaphthalenium halides 37 in 79% yield"
+      ner = OSCAR4.new
+      str  = "Alternatively, rearrangement of O-(w-haloalkyl)esters 34 of 2-carboethoxy-N-hydroxypyridine-2-selone affords azonianaphthalenium halides 37 in 79% yield"
 
       mentions = ner.match(str, "CM", false)
-      good_mentions = ["2-carboethoxy-N-hydroxypyridine-2-selone", "O-(ω-haloalkyl)esters"]
+      good_mentions = ["2-carboethoxy-N-hydroxypyridine-2-selone", "O-(w-haloalkyl)esters"]
 
       good_mentions.each{|mention| 
         assert(mentions.include? mention)
@@ -25,18 +24,18 @@ class TestOSCAR3 < Test::Unit::TestCase
 
   def test_ranges
     begin
-      ner = OSCAR3.new
+      ner = OSCAR4.new
       str =<<-EOF 
 This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
 This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
 This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
 This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
-This otherone talks about O-(ω-haloalkyl)esters.
-This otherone talks about O-(ω-haloalkyl)esters.
-This otherone talks about O-(ω-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
 
-This otherone talks about O-(ω-haloalkyl)esters.
-This otherone talks about O-(ω-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
+This otherone talks about O-(w-haloalkyl)esters.
       EOF
 
 

data/test/rbbt/ner/test_patterns.rb

@@ -0,0 +1,66 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
+require 'rbbt/ner/patterns'
+
+class TestPatternRelExt < Test::Unit::TestCase
+  def test_simple_pattern
+    text = "Experiments have shown that TP53 interacts with CDK5 under certain conditions"
+
+    gene1 = "TP53"
+    NamedEntity.annotate(gene1, text.index(gene1), "Gene")
+
+    gene2 = "CDK5"
+    NamedEntity.annotate(gene2, text.index(gene2), "Gene")
+
+    interaction = "interacts"
+    NamedEntity.annotate(interaction, text.index(interaction), "Interaction")
+
+    Annotated.annotate(text, [gene1, gene2, interaction])
+
+    assert_equal "TP53 interacts with CDK5", PatternRelExt.simple_pattern(text, "GENE INTERACTION with GENE").first
+
+  end
+
+  def test_chunk_pattern
+    text = "Experiments have shown that TP53 found in cultivated cells interacts with CDK5 under certain conditions"
+
+    gene1 = "TP53"
+    NamedEntity.annotate(gene1, text.index(gene1), "Gene")
+
+    gene2 = "CDK5"
+    NamedEntity.annotate(gene2, text.index(gene2), "Gene")
+
+    interaction = "interacts"
+    NamedEntity.annotate(interaction, text.index(interaction), "Interaction")
+
+    Annotated.annotate(text, {:entities => [gene1, gene2, interaction]})
+
+    assert_equal "TP53 found in cultivated cells interacts with CDK5", 
+      PatternRelExt.new("NP[entity:Gene] VP[stem:interacts] with NP[entity:Gene]").match_sentences([text]).first.first
+
+    assert_equal "TP53 found in cultivated cells interacts with CDK5", 
+      PatternRelExt.new(["NP[entity:Gene] VP[stem:interacts] with NP[entity:Gene]"]).match_sentences([text]).first.first
+  end
+
+  def test_chunk_pattern
+    text = "There is a concern with the use of thiazolidinediones in patients with an increased risk of colon cancer (e.g., familial colon polyposis)."
+
+    drug = "thiazolidinediones"
+    NamedEntity.annotate(drug, text.index(drug), "Chemical Mention")
+
+    disease = "colon cancer"
+    NamedEntity.annotate(disease, text.index(disease), "disease")
+
+    Annotated.annotate(text, {:entitites => [drug, disease]})
+
+    assert_equal "thiazolidinediones in patients with an increased risk of colon cancer", 
+      PatternRelExt.new("NP[entity:Chemical Mention] NP[stem:risk] NP[entity:disease]").match_sentences([text]).first.first
+
+  end
+
+
+  def test_entities_with_spaces
+    PatternRelExt.new("NP[entity:Gene Name]").token_trie
+  end
+
+
+end