rbbt-text 0.5.0 → 0.6.0

data/test/rbbt/ner/segment/test_named_entity.rb

@@ -0,0 +1,29 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/ner/segment'
+require 'rbbt/ner/segment/named_entity'
+
+class TestClass < Test::Unit::TestCase
+  def test_info
+    a = "test"
+    NamedEntity.setup a
+    assert(! a.info.keys.include?(:code))
+    a.code = 10
+    a.offset = 100
+    assert a.info.include? :code
+    assert a.info.include? :offset
+  end
+
+  def test_all_args
+    a = "test"
+    NamedEntity.setup a, 10, "TYPE", "CODE", "SCORE"
+    assert_equal 10, a.offset
+  end
+
+  def test_tsv
+    a = "test"
+    NamedEntity.setup a, 10, "TYPE", "CODE", "SCORE"
+    assert Segment.tsv([a]).fields.include? "code"
+    assert Segment.tsv([a], nil).fields.include? "code"
+    assert Segment.tsv([a], "literal").fields.include? "code"
+  end
+end

data/test/rbbt/ner/segment/test_segmented.rb

@@ -0,0 +1,23 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/ner/segment/segmented'
+
+class TestClass < Test::Unit::TestCase
+  def test_split
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = a.index gene1
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = a.index gene2
+
+    gene3 = "TP53 gene"
+    gene3.extend Segment
+    gene3.offset = a.index gene3
+
+    Segmented.setup(a, [gene2, gene1, gene3])
+    assert_equal ["This sentence mentions the ", gene3, " and the ", gene2, " protein"], a.split_segments
+  end
+end

data/test/rbbt/ner/{annotations → segment}/test_transformed.rb

@@ -1,6 +1,6 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/ner/annotations/transformed'
-require 'rbbt/ner/annotations/named_entity'
+require 'rbbt/ner/segment/transformed'
+require 'rbbt/ner/segment/named_entity'
 
 class TestClass < Test::Unit::TestCase
   def test_transform
@@ -8,11 +8,11 @@ class TestClass < Test::Unit::TestCase
     original = a.dup
 
     gene1 = "TP53"
-    gene1.extend NamedEntity
+    gene1.extend Segment
     gene1.offset = a.index gene1
 
     gene2 = "CDK5"
-    gene2.extend NamedEntity
+    gene2.extend Segment
     gene2.offset = a.index gene2
 
     assert_equal gene1, a[gene1.range]
@@ -30,7 +30,7 @@ class TestClass < Test::Unit::TestCase
 
 
     gene3 = "GN gene"
-    gene3.extend NamedEntity
+    gene3.extend Segment
     gene3.offset = a.index gene3
 
     assert_equal gene3, a[gene3.range]
@@ -108,7 +108,7 @@ class TestClass < Test::Unit::TestCase
 
   def test_html_with_offset
     a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
-    Segment.annotate(a, 10)
+    Segment.setup(a, 10)
 
     gene1 = "TP53"
     gene1.extend NamedEntity

data/test/rbbt/ner/test_ngram_prefix_dictionary.rb

@@ -10,7 +10,21 @@ C2;11;22;3 3;bb
     EOF
 
     TmpFile.with_file(lexicon) do |file|
-      index = NGramPrefixDictionary.new(TSV.new(file, :flat, :sep => ';'), "test")
+      index = NGramPrefixDictionary.new(TSV.open(file, :flat, :sep => ';'), "test")
+
+      matches = index.match(' asdfa dsf asdf aa asdfasdf ')
+      assert matches.select{|m| m.code.include? 'C1'}.any?
+    end
+  end
+
+  def test_stream
+    lexicon =<<-EOF
+C1;aa;AA;bb b
+C2;11;22;3 3;bb
+    EOF
+
+    TmpFile.with_file(lexicon.gsub(/;/,"\t")) do |file|
+      index = NGramPrefixDictionary.new(file, "test")
 
       matches = index.match(' asdfa dsf asdf aa asdfasdf ')
       assert matches.select{|m| m.code.include? 'C1'}.any?

data/test/rbbt/ner/test_patterns.rb

@@ -6,33 +6,32 @@ class TestPatternRelExt < Test::Unit::TestCase
     text = "Experiments have shown that TP53 interacts with CDK5 under certain conditions"
 
     gene1 = "TP53"
-    NamedEntity.annotate(gene1, text.index(gene1), "Gene")
+    NamedEntity.setup(gene1, text.index(gene1), "Gene")
 
     gene2 = "CDK5"
-    NamedEntity.annotate(gene2, text.index(gene2), "Gene")
+    NamedEntity.setup(gene2, text.index(gene2), "Gene")
 
     interaction = "interacts"
-    NamedEntity.annotate(interaction, text.index(interaction), "Interaction")
+    NamedEntity.setup(interaction, text.index(interaction), "Interaction")
 
-    Annotated.annotate(text, [gene1, gene2, interaction])
+    Segmented.setup(text, [gene1, gene2, interaction])
 
     assert_equal "TP53 interacts with CDK5", PatternRelExt.simple_pattern(text, "GENE INTERACTION with GENE").first
-
   end
 
   def test_chunk_pattern
     text = "Experiments have shown that TP53 found in cultivated cells interacts with CDK5 under certain conditions"
 
     gene1 = "TP53"
-    NamedEntity.annotate(gene1, text.index(gene1), "Gene")
+    NamedEntity.setup(gene1, text.index(gene1), "Gene")
 
     gene2 = "CDK5"
-    NamedEntity.annotate(gene2, text.index(gene2), "Gene")
+    NamedEntity.setup(gene2, text.index(gene2), "Gene")
 
     interaction = "interacts"
-    NamedEntity.annotate(interaction, text.index(interaction), "Interaction")
+    NamedEntity.setup(interaction, text.index(interaction), "Interaction")
 
-    Annotated.annotate(text, {:entities => [gene1, gene2, interaction]})
+    Segmented.setup(text, {:entities => [gene1, gene2, interaction]})
 
     assert_equal "TP53 found in cultivated cells interacts with CDK5", 
       PatternRelExt.new("NP[entity:Gene] VP[stem:interacts] with NP[entity:Gene]").match_sentences([text]).first.first
@@ -45,12 +44,12 @@ class TestPatternRelExt < Test::Unit::TestCase
     text = "There is a concern with the use of thiazolidinediones in patients with an increased risk of colon cancer (e.g., familial colon polyposis)."
 
     drug = "thiazolidinediones"
-    NamedEntity.annotate(drug, text.index(drug), "Chemical Mention")
+    NamedEntity.setup(drug, text.index(drug), "Chemical Mention")
 
     disease = "colon cancer"
-    NamedEntity.annotate(disease, text.index(disease), "disease")
+    NamedEntity.setup(disease, text.index(disease), "disease")
 
-    Annotated.annotate(text, {:entitites => [drug, disease]})
+    Segmented.setup(text, {:entitites => [drug, disease]})
 
     assert_equal "thiazolidinediones in patients with an increased risk of colon cancer", 
       PatternRelExt.new("NP[entity:Chemical Mention] NP[stem:risk] NP[entity:disease]").match_sentences([text]).first.first

data/test/rbbt/ner/test_regexpNER.rb

@@ -55,12 +55,13 @@ class TestRegExpNER < Test::Unit::TestCase
     assert_equal "In this sentence I should find ".length, matches.select{|m| m.type == :this}[1].offset
     assert_equal :this, matches.select{|m| m.type == :this}[0].type
 
-    Annotated.annotate(sentence)
+    Segmented.setup(sentence)
     ner_this = RegExpNER.new({:this => /this/})
     ner_that = RegExpNER.new({:that => /that/})
-    sentence.annotations += ner_this.entities(sentence)
-    sentence.annotations += ner_that.entities(sentence)
-    matches = sentence.annotations
+    sentence.segments ||= []
+    sentence.segments += ner_this.entities(sentence)
+    sentence.segments += ner_that.entities(sentence)
+    matches = sentence.segments
 
     assert_equal ["this", "this", "that"].sort, matches.sort
     assert_equal "In ".length, matches.select{|m| m.type == :this}[0].offset

data/test/rbbt/ner/test_segment.rb

@@ -0,0 +1,101 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/ner/segment'
+
+class TestClass < Test::Unit::TestCase
+  def test_info
+    a = "test"
+    a.extend Segment
+    a.offset = 10
+    assert a.info.include? :offset
+  end
+
+  def test_sort
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = a.index gene1
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = a.index gene2
+
+    assert_equal [gene1,gene2], Segment.sort([gene2,gene1])
+  end
+
+  def test_clean_sort
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = a.index gene1
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = a.index gene2
+
+    gene3 = "TP53 gene"
+    gene3.extend Segment
+    gene3.offset = a.index gene3
+
+    assert_equal [gene3,gene2], Segment.clean_sort([gene2,gene1,gene3])
+  end
+
+  def test_split
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = a.index gene1
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = a.index gene2
+
+    gene3 = "TP53 gene"
+    gene3.extend Segment
+    gene3.offset = a.index gene3
+
+    assert_equal ["This sentence mentions the ", gene3, " and the ", gene2, " protein"], Segment.split(a, [gene2,gene1,gene3])
+  end
+
+
+  def test_align
+    text =<<-EOF
+Atypical teratoid/rhabdoid tumors (AT/RTs) are highly aggressive brain tumors of early childhood poorly responding to therapy.
+    EOF
+
+    parts = text.split(/\W/)
+    Segment.align(text, parts)
+
+    assert_equal "Atypical teratoid/".length, parts.select{|s| s == "rhabdoid"}.first.offset
+  end
+
+  def test_segment_index
+    a = "This sentence mentions the TP53 gene and the CDK5R1 protein"
+
+    gene1 = "TP53"
+    gene1.extend Segment
+    gene1.offset = a.index gene1
+
+    gene2 = "CDK5R1"
+    gene2.extend Segment
+    gene2.offset = a.index gene2
+
+    gene3 = "TP53 gene"
+    gene3.extend Segment
+    gene3.offset = a.index gene3
+
+    index = Segment.index([gene1, gene2, gene3])
+    assert_equal %w(CDK5R1), index[gene2.offset + 1]
+
+    TmpFile.with_file do |fwt|
+      index = Segment.index([gene1, gene2, gene3], fwt)
+      assert_equal %w(CDK5R1), index[gene2.offset + 1]
+      index = Segment.index([gene1, gene2, gene3], fwt)
+      assert_equal %w(CDK5R1), index[gene2.offset + 1]
+    end
+  end
+
+end
+

data/test/rbbt/ner/test_token_trieNER.rb

@@ -30,7 +30,7 @@ C2;11;22;3 3;bb
 
     TmpFile.with_file(lexicon) do |file|
 
-      index = TokenTrieNER.process({}, TSV.new(file, :flat, :sep => ';'))
+      index = TokenTrieNER.process({}, TSV.open(file, :flat, :sep => ';'))
 
       assert_equal ['AA', 'aa', 'bb', '11', '22', '3'].sort, index.keys.sort
       assert_equal [:END], index['aa'].keys
@@ -47,7 +47,7 @@ C2;11;22;3 3;bb
 
 
     TmpFile.with_file(lexicon) do |file|
-      index = TokenTrieNER.process({}, TSV.new(file, :sep => ';', :type => :flat ))
+      index = TokenTrieNER.process({}, TSV.open(file, :sep => ';', :type => :flat ))
 
       assert TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf').extend(TokenTrieNER::EnumeratedArray), false).first.collect{|c| c.code}.include?   'C1'
       assert_equal %w(aa), TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf').extend(TokenTrieNER::EnumeratedArray), false).last
@@ -71,9 +71,8 @@ C2;11;22;3 3;bb
     EOF
 
     TmpFile.with_file(lexicon) do |file|
-      index = TokenTrieNER.new("test", TSV.new(file, :flat, :sep => ';'))
+      index = TokenTrieNER.new("test", TSV.open(file, :flat, :sep => ';'))
 
-      index.match(' asdfa dsf asdf aa asdfasdf ')
       assert index.match(' asdfa dsf asdf aa asdfasdf ').select{|m| m.code.include? 'C1'}.any?
     end
   end
@@ -81,18 +80,19 @@ C2;11;22;3 3;bb
   def test_slack
     lexicon =<<-EOF
 C1;aa;AA;bb cc cc b
-C2;11;22;3 3;bb
+C2;11;22;3 3;bb;bbbb
     EOF
 
     TmpFile.with_file(lexicon) do |file|
       index = TokenTrieNER.new({})
       index.slack = Proc.new{|t| t =~ /^c*$/}
 
-      index.merge TSV.new(file, :flat, :sep => ';')
+      index.merge TSV.open(file, :flat, :sep => ';')
 
       assert index.match(' aaaaa 3 cc 3').select{|m| m.code.include? 'C2'}.any?
       assert index.match(' bb cc b').select{|m| m.code.include? 'C1'}.any?
       assert index.match(' bb b').select{|m| m.code.include? 'C1'}.any?
+      assert index.match(' BBBB b').select{|m| m.code.include? 'C2'}.any?
     end
   end
 
@@ -106,7 +106,7 @@ C2;11;22;3 3;bb
       index = TokenTrieNER.new({})
       index.slack = Proc.new{|t| t =~ /^c*$/}
 
-      index.merge TSV.new(file, :flat, :sep => ';')
+      index.merge TSV.open(file, :flat, :sep => ';')
 
       assert index.match(Token.tokenize('3 cc 3')).select{|m| m.code.include? 'C2'}.any?
     end
@@ -126,9 +126,8 @@ C2;11;22;3 3;bb
     EOF
 
     TmpFile.with_file(lexicon) do |file|
-      index = TokenTrieNER.new("test", TSV.new(file, :flat, :sep => ';'), :persistence => true)
+      index = TokenTrieNER.new("test", TSV.open(file, :flat, :sep => ';'), :persistence => true)
 
-      index.match(' asdfa dsf asdf aa asdfasdf ')
       assert index.match(' asdfa dsf asdf aa asdfasdf ').select{|m| m.code.include? 'C1'}.any?
     end
   end

data/test/test_helper.rb

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
 require 'rbbt'
-require 'rbbt/util/persistence'
+require 'rbbt/persist'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/log'
 require 'rbbt/corpus/document_repo'
@@ -15,15 +15,15 @@ class Test::Unit::TestCase
 
   def setup
     FileUtils.mkdir_p Rbbt.tmp.test.persistence.find(:user)
-    Persistence.cachedir = Rbbt.tmp.test.persistence.find :user
+    Persist.cachedir = Rbbt.tmp.test.persistence.find :user
   end
 
   def teardown
     FileUtils.rm_rf Rbbt.tmp.test.find :user
-    TCHash::CONNECTIONS.values.each do |c| c.close end
-    TCHash::CONNECTIONS.clear
-    DocumentRepo::CONNECTIONS.values.each do |c| c.close end
-    DocumentRepo::CONNECTIONS.clear
+    Persist::TC_CONNECTIONS.values.each do |c| c.close end
+    Persist::TC_CONNECTIONS.clear
+    DocumentRepo::TC_CONNECTIONS.values.each do |c| c.close end
+    DocumentRepo::TC_CONNECTIONS.clear
   end
 
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-text
 version: !ruby/object:Gem::Version 
-  hash: 11
-  prerelease: 
+  hash: 7
+  prerelease: false
   segments: 
   - 0
-  - 5
+  - 6
   - 0
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-05 00:00:00 +02:00
+date: 2011-09-07 00:00:00 +02:00
 default_executable: get_ppis.rb
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,10 +26,12 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 63
         segments: 
+        - 4
         - 0
-        version: "0"
+        - 0
+        version: 4.0.0
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -74,6 +76,20 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rjb
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
 description: "Text mining tools: named entity recognition and normalization, document classification, bag-of-words, dictionaries, etc"
 email: miguel.vazquez@fdi.ucm.es
 executables: 
@@ -92,12 +108,6 @@ files:
 - lib/rbbt/corpus/sources/pubmed.rb
 - lib/rbbt/ner/NER.rb
 - lib/rbbt/ner/abner.rb
-- lib/rbbt/ner/annotations.rb
-- lib/rbbt/ner/annotations/annotated.rb
-- lib/rbbt/ner/annotations/named_entity.rb
-- lib/rbbt/ner/annotations/relations.rb
-- lib/rbbt/ner/annotations/token.rb
-- lib/rbbt/ner/annotations/transformed.rb
 - lib/rbbt/ner/banner.rb
 - lib/rbbt/ner/chemical_tagger.rb
 - lib/rbbt/ner/ngram_prefix_dictionary.rb
@@ -108,6 +118,12 @@ files:
 - lib/rbbt/ner/rnorm.rb
 - lib/rbbt/ner/rnorm/cue_index.rb
 - lib/rbbt/ner/rnorm/tokens.rb
+- lib/rbbt/ner/segment.rb
+- lib/rbbt/ner/segment/named_entity.rb
+- lib/rbbt/ner/segment/relationship.rb
+- lib/rbbt/ner/segment/segmented.rb
+- lib/rbbt/ner/segment/token.rb
+- lib/rbbt/ner/segment/transformed.rb
 - lib/rbbt/ner/token_trieNER.rb
 - lib/rbbt/nlp/genia/sentence_splitter.rb
 - lib/rbbt/nlp/nlp.rb
@@ -130,12 +146,13 @@ files:
 - test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_abner.rb
 - test/rbbt/ner/test_banner.rb
-- test/rbbt/ner/annotations/test_transformed.rb
-- test/rbbt/ner/annotations/test_named_entity.rb
+- test/rbbt/ner/test_NER.rb
 - test/rbbt/ner/test_token_trieNER.rb
-- test/rbbt/ner/test_annotations.rb
 - test/rbbt/ner/test_patterns.rb
-- test/rbbt/ner/test_NER.rb
+- test/rbbt/ner/segment/test_named_entity.rb
+- test/rbbt/ner/segment/test_segmented.rb
+- test/rbbt/ner/segment/test_transformed.rb
+- test/rbbt/ner/test_segment.rb
 - test/rbbt/ner/test_rnorm.rb
 - test/rbbt/ner/test_oscar4.rb
 - test/rbbt/ner/test_chemical_tagger.rb
@@ -174,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Text mining tools for the Ruby Bioinformatics Toolkit (rbbt)
@@ -186,12 +203,13 @@ test_files:
 - test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_abner.rb
 - test/rbbt/ner/test_banner.rb
-- test/rbbt/ner/annotations/test_transformed.rb
-- test/rbbt/ner/annotations/test_named_entity.rb
+- test/rbbt/ner/test_NER.rb
 - test/rbbt/ner/test_token_trieNER.rb
-- test/rbbt/ner/test_annotations.rb
 - test/rbbt/ner/test_patterns.rb
-- test/rbbt/ner/test_NER.rb
+- test/rbbt/ner/segment/test_named_entity.rb
+- test/rbbt/ner/segment/test_segmented.rb
+- test/rbbt/ner/segment/test_transformed.rb
+- test/rbbt/ner/test_segment.rb
 - test/rbbt/ner/test_rnorm.rb
 - test/rbbt/ner/test_oscar4.rb
 - test/rbbt/ner/test_chemical_tagger.rb